test(generator): усилены проверки startup-history и Superset

- Зачем: - зелёный результат проверок должен означать фактический стык и точный контракт дашборда. - Что: - проверка startup-history читает manifest, требует live seam и непустой ODS. - добавлен быстрый runtime gate для daily-wave и live-продолжения. - Superset sync ограничен целевым dashboard и сверяет существенные params. - Проверка: - docker target tests; make generator-test; make generated-history-runtime-check.
2026-07-05 21:32:51 +03:00
parent 9804d0beae
commit 00c97eb063
11 changed files with 500 additions and 23 deletions
@@ -12,8 +12,9 @@ CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-123456}"
 GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
 GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
 REQUIRE_SUPERSET="${REQUIRE_SUPERSET:-1}"
-CHECK_LIVE_SEAM="${CHECK_LIVE_SEAM:-auto}"
+CHECK_LIVE_SEAM="${CHECK_LIVE_SEAM:-1}"
 GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES:-10}"
+WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS:-10}"

 fail() {
  echo "Ошибка: $*" >&2
@@ -50,11 +51,39 @@ CH_MODEL_T_END="$(clickhouse_datetime_literal "${GEN_MODEL_T_END}")"

 [[ "${GEN_LIVE_CHECK_MINUTES}" =~ ^[0-9]+$ ]] \
  || fail "GEN_LIVE_CHECK_MINUTES должен быть целым числом минут"
+[[ "${WAIT_LIVE_ROWS_SECONDS}" =~ ^[0-9]+$ ]] \
+  || fail "WAIT_LIVE_ROWS_SECONDS должен быть целым числом секунд"
 case "${CHECK_LIVE_SEAM}" in
  0|1|auto) ;;
  *) fail "CHECK_LIVE_SEAM должен быть 0, 1 или auto" ;;
 esac

+echo "=== Проверка startup-history manifest ==="
+
+manifest_summary="$(${COMPOSE_BIN} run --rm --no-deps \
+  generator python -m clickstream_generator.startup_history_artifact kafka-manifest-summary)"
+
+IFS=$'\t' read -r manifest_events manifest_visits manifest_users manifest_min_ts manifest_max_ts manifest_model_t0 manifest_model_t_end manifest_launch_profile <<< "${manifest_summary}"
+
+[[ "${manifest_events}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать startup-history manifest из Kafka"
+[[ -n "${manifest_model_t0}" ]] || fail "startup-history manifest не содержит model_t0"
+[[ -n "${manifest_model_t_end}" ]] || fail "startup-history manifest не содержит model_t_end"
+
+CH_MODEL_T0="$(clickhouse_datetime_literal "${manifest_model_t0}")"
+CH_MODEL_T_END="$(clickhouse_datetime_literal "${manifest_model_t_end}")"
+
+expected_profile="${PROFILE:-${GEN_LAUNCH_PROFILE:-}}"
+if [[ -n "${expected_profile}" && -n "${manifest_launch_profile}" && "${expected_profile}" != "${manifest_launch_profile}" ]]; then
+  fail "проверка запущена для профиля ${expected_profile}, но manifest от ${manifest_launch_profile}"
+fi
+
+echo "manifest_events=${manifest_events}"
+echo "manifest_visits=${manifest_visits}"
+echo "manifest_users=${manifest_users}"
+echo "manifest_model_t0=${manifest_model_t0}"
+echo "manifest_model_t_end=${manifest_model_t_end}"
+echo "manifest_launch_profile=${manifest_launch_profile}"
+
 echo "=== Проверка ClickHouse: данные генерации в DM ==="

 stats_query="
@@ -271,7 +300,7 @@ echo "monotonic_ok=${contains_monotonic_ok}"
 echo "confirmation_share=${contains_confirmation_share}"

 should_check_live_seam="${CHECK_LIVE_SEAM}"
-if [[ "${should_check_live_seam}" == "auto" ]]; then
+if [[ "${should_check_live_seam}" != "0" ]]; then
  live_rows_query="
 WITH
    toDateTime64('${CH_MODEL_T_END}', 6) AS t_end,
@@ -280,12 +309,25 @@ SELECT count()
 FROM dds.event
 WHERE event_ts >= t_end AND event_ts < t_live_end
 FORMAT TabSeparated"
-  live_rows="$(ch_query "${live_rows_query}")"
-  [[ "${live_rows}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать live-строки DDS после GEN_MODEL_T_END"
-  if (( live_rows > 0 )); then
-    should_check_live_seam="1"
-  else
-    should_check_live_seam="0"
+  live_rows=0
+  deadline=$((SECONDS + WAIT_LIVE_ROWS_SECONDS))
+  while true; do
+    live_rows="$(ch_query "${live_rows_query}")"
+    [[ "${live_rows}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать live-строки DDS после GEN_MODEL_T_END"
+    if (( live_rows > 0 || SECONDS >= deadline )); then
+      break
+    fi
+    sleep 1
+  done
+  if [[ "${should_check_live_seam}" == "1" ]] && (( live_rows == 0 )); then
+    fail "live-продолжение не записало строки в первые ${GEN_LIVE_CHECK_MINUTES} минут после manifest model_t_end"
+  fi
+  if [[ "${CHECK_LIVE_SEAM}" == "auto" ]]; then
+    if (( live_rows > 0 )); then
+      should_check_live_seam="1"
+    else
+      should_check_live_seam="0"
+    fi
  fi
 fi

@@ -425,19 +467,35 @@ WITH
    )
 SELECT
    (SELECT count() FROM crossing) AS crossing_visits,
+    (
+      SELECT count()
+      FROM ods.device_by_click AS d
+      INNER JOIN crossing AS c ON c.click_id = d.click_id
+      WHERE d.click_id IS NOT NULL
+    ) AS ods_device_rows,
+    (
+      SELECT count()
+      FROM ods.geo_by_click AS g
+      INNER JOIN crossing AS c ON c.click_id = g.click_id
+      WHERE g.click_id IS NOT NULL
+    ) AS ods_geo_rows,
    (SELECT count() FROM device_conflicts) AS ods_device_conflicts,
    (SELECT count() FROM geo_conflicts) AS ods_geo_conflicts
 FORMAT TabSeparated"

  ods_context="$(ch_query "${ods_context_query}")"
-  IFS=$'\t' read -r ods_crossing_visits ods_device_conflicts ods_geo_conflicts <<< "${ods_context}"
+  IFS=$'\t' read -r ods_crossing_visits ods_device_rows ods_geo_rows ods_device_conflicts ods_geo_conflicts <<< "${ods_context}"

  [[ "${ods_crossing_visits}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать ODS-проверку стыка"
  (( ods_crossing_visits == crossing_visits )) \
    || fail "ODS и DDS нашли разное число переходящих визитов: ${ods_crossing_visits}/${crossing_visits}"
+  (( ods_device_rows > 0 )) || fail "ODS device пустой на стыке"
+  (( ods_geo_rows > 0 )) || fail "ODS geo пустой на стыке"
  (( ods_device_conflicts == 0 )) || fail "ODS device конфликтует на стыке: ${ods_device_conflicts}"
  (( ods_geo_conflicts == 0 )) || fail "ODS geo конфликтует на стыке: ${ods_geo_conflicts}"

+  echo "ods_device_rows=${ods_device_rows}"
+  echo "ods_geo_rows=${ods_geo_rows}"
  echo "ods_device_conflicts=${ods_device_conflicts}"
  echo "ods_geo_conflicts=${ods_geo_conflicts}"
 else
@@ -105,6 +105,7 @@ ${COMPOSE_BIN} up -d --no-deps superset
 echo "Шаг 8: техническая проверка аналитического контура"
 GEN_MODEL_T0="${GEN_MODEL_T0}" \
 GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
+CHECK_LIVE_SEAM=0 \
 COMPOSE_BIN="${COMPOSE_BIN}" \
 bash "${SCRIPT_DIR}/check_generated_analytics.sh"

@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+#
+# Быстрый runtime gate для issue 17: проверяет, что daily-wave доходит до
+# manifest/check, а live-продолжение реально записывает стык backfill/live.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
+PROFILE="${PROFILE:-daily-wave}"
+GEN_HISTORY_DURATION="${GEN_HISTORY_DURATION:-1h}"
+WAIT_CLICKHOUSE_SECONDS="${WAIT_CLICKHOUSE_SECONDS:-60}"
+WAIT_STG_SECONDS="${WAIT_STG_SECONDS:-5}"
+LIVE_SECONDS="${LIVE_SECONDS:-25}"
+GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES:-10}"
+WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS:-2}"
+
+cd "${REPO_ROOT}"
+
+launch_env="$(
+  PYTHONPATH="${REPO_ROOT}/generator/src" \
+  uv run python -m clickstream_generator.launch \
+    backfill \
+    --profile "${PROFILE}" \
+    --duration "${GEN_HISTORY_DURATION}"
+)"
+eval "${launch_env}"
+
+wait_for_clickhouse() {
+  local deadline
+  deadline=$((SECONDS + WAIT_CLICKHOUSE_SECONDS))
+
+  until ${COMPOSE_BIN} exec -T clickhouse clickhouse-client \
+      --user=default \
+      --password=123456 \
+      --query "SELECT 1" >/dev/null 2>&1; do
+    if (( SECONDS >= deadline )); then
+      echo "Ошибка: ClickHouse не ответил за ${WAIT_CLICKHOUSE_SECONDS} сек." >&2
+      exit 1
+    fi
+    sleep 2
+  done
+}
+
+stg_total_rows() {
+  ${COMPOSE_BIN} exec -T clickhouse clickhouse-client \
+    --user=default \
+    --password=123456 \
+    --query "
+SELECT
+    (SELECT count() FROM stg.browser_raw)
+  + (SELECT count() FROM stg.location_raw)
+  + (SELECT count() FROM stg.device_raw)
+  + (SELECT count() FROM stg.geo_raw)
+FORMAT TabSeparated"
+}
+
+cleanup_live_generator() {
+  ${COMPOSE_BIN} stop generator >/dev/null 2>&1 || true
+}
+trap cleanup_live_generator EXIT
+
+echo "=== Быстрая runtime-проверка startup-history/live seam ==="
+echo "GEN_LAUNCH_PROFILE=${GEN_LAUNCH_PROFILE}"
+echo "GEN_HISTORY_DURATION=${GEN_HISTORY_DURATION}"
+echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
+echo ""
+
+echo "Шаг 0: очистка volumes ClickHouse/Kafka/state"
+${COMPOSE_BIN} --profile live-generator down -v --remove-orphans
+
+echo "Шаг 1: запуск ClickHouse и Kafka"
+${COMPOSE_BIN} up -d clickhouse kafka
+wait_for_clickhouse
+
+echo "Шаг 2: применение DDL"
+bash "${SCRIPT_DIR}/apply_clickhouse_ddl.sh"
+
+echo "Шаг 3: сборка образа генератора"
+${COMPOSE_BIN} build generator
+
+echo "Шаг 4: короткий backfill стартовой истории"
+${COMPOSE_BIN} run --rm --no-deps \
+  -e GEN_RUN_MODE=backfill \
+  -e GEN_STATE_RESET=true \
+  -e GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
+  -e GEN_SEED="${GEN_SEED}" \
+  -e GEN_MODEL_T0="${GEN_MODEL_T0}" \
+  -e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
+  -e GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
+  -e GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
+  -e GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
+  -e GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
+  -e GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
+  -e GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
+  -e GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
+  generator
+
+echo "Шаг 5: первый batch STG -> ODS -> DDS -> DM"
+sleep "${WAIT_STG_SECONDS}"
+bash "${SCRIPT_DIR}/run_batch.sh"
+stg_rows_before_live="$(stg_total_rows)"
+
+echo "Шаг 6: live-продолжение, ждём новые STG-строки до ${LIVE_SECONDS} сек."
+GEN_RUN_MODE=live \
+GEN_STATE_RESET=false \
+GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
+GEN_SEED="${GEN_SEED}" \
+GEN_MODEL_T0="${GEN_MODEL_T0}" \
+GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
+GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
+GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
+GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
+GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
+GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
+GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
+GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
+${COMPOSE_BIN} up -d --build generator
+deadline=$((SECONDS + LIVE_SECONDS))
+while true; do
+  stg_rows_after_live="$(stg_total_rows)"
+  if (( stg_rows_after_live > stg_rows_before_live )); then
+    echo "live_stg_rows_before=${stg_rows_before_live}"
+    echo "live_stg_rows_after=${stg_rows_after_live}"
+    break
+  fi
+  if (( SECONDS >= deadline )); then
+    echo "Ошибка: live-продолжение не записало новые STG-строки за ${LIVE_SECONDS} сек." >&2
+    exit 1
+  fi
+  sleep 1
+done
+cleanup_live_generator
+
+echo "Шаг 7: второй batch после live"
+sleep "${WAIT_STG_SECONDS}"
+bash "${SCRIPT_DIR}/run_batch.sh"
+
+echo "Шаг 8: проверка manifest/profile/live seam без Superset"
+PROFILE="${PROFILE}" \
+GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
+GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES}" \
+WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS}" \
+CHECK_LIVE_SEAM=1 \
+REQUIRE_SUPERSET=0 \
+COMPOSE_BIN="${COMPOSE_BIN}" \
+bash "${SCRIPT_DIR}/check_generated_analytics.sh"
+
+echo ""
+echo "Готово: runtime-проверка startup-history/live seam прошла."