ddadmin
00c97eb063
- Зачем: - зелёный результат проверок должен означать фактический стык и точный контракт дашборда. - Что: - проверка startup-history читает manifest, требует live seam и непустой ODS. - добавлен быстрый runtime gate для daily-wave и live-продолжения. - Superset sync ограничен целевым dashboard и сверяет существенные params. - Проверка: - docker target tests; make generator-test; make generated-history-runtime-check.
153 lines
5.0 KiB
Bash
153 lines
5.0 KiB
Bash
#!/usr/bin/env bash
|
||
#
|
||
# Быстрый runtime gate для issue 17: проверяет, что daily-wave доходит до
|
||
# manifest/check, а live-продолжение реально записывает стык backfill/live.
|
||
|
||
set -euo pipefail
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||
|
||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||
PROFILE="${PROFILE:-daily-wave}"
|
||
GEN_HISTORY_DURATION="${GEN_HISTORY_DURATION:-1h}"
|
||
WAIT_CLICKHOUSE_SECONDS="${WAIT_CLICKHOUSE_SECONDS:-60}"
|
||
WAIT_STG_SECONDS="${WAIT_STG_SECONDS:-5}"
|
||
LIVE_SECONDS="${LIVE_SECONDS:-25}"
|
||
GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES:-10}"
|
||
WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS:-2}"
|
||
|
||
cd "${REPO_ROOT}"
|
||
|
||
launch_env="$(
|
||
PYTHONPATH="${REPO_ROOT}/generator/src" \
|
||
uv run python -m clickstream_generator.launch \
|
||
backfill \
|
||
--profile "${PROFILE}" \
|
||
--duration "${GEN_HISTORY_DURATION}"
|
||
)"
|
||
eval "${launch_env}"
|
||
|
||
wait_for_clickhouse() {
|
||
local deadline
|
||
deadline=$((SECONDS + WAIT_CLICKHOUSE_SECONDS))
|
||
|
||
until ${COMPOSE_BIN} exec -T clickhouse clickhouse-client \
|
||
--user=default \
|
||
--password=123456 \
|
||
--query "SELECT 1" >/dev/null 2>&1; do
|
||
if (( SECONDS >= deadline )); then
|
||
echo "Ошибка: ClickHouse не ответил за ${WAIT_CLICKHOUSE_SECONDS} сек." >&2
|
||
exit 1
|
||
fi
|
||
sleep 2
|
||
done
|
||
}
|
||
|
||
stg_total_rows() {
|
||
${COMPOSE_BIN} exec -T clickhouse clickhouse-client \
|
||
--user=default \
|
||
--password=123456 \
|
||
--query "
|
||
SELECT
|
||
(SELECT count() FROM stg.browser_raw)
|
||
+ (SELECT count() FROM stg.location_raw)
|
||
+ (SELECT count() FROM stg.device_raw)
|
||
+ (SELECT count() FROM stg.geo_raw)
|
||
FORMAT TabSeparated"
|
||
}
|
||
|
||
cleanup_live_generator() {
|
||
${COMPOSE_BIN} stop generator >/dev/null 2>&1 || true
|
||
}
|
||
trap cleanup_live_generator EXIT
|
||
|
||
echo "=== Быстрая runtime-проверка startup-history/live seam ==="
|
||
echo "GEN_LAUNCH_PROFILE=${GEN_LAUNCH_PROFILE}"
|
||
echo "GEN_HISTORY_DURATION=${GEN_HISTORY_DURATION}"
|
||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||
echo ""
|
||
|
||
echo "Шаг 0: очистка volumes ClickHouse/Kafka/state"
|
||
${COMPOSE_BIN} --profile live-generator down -v --remove-orphans
|
||
|
||
echo "Шаг 1: запуск ClickHouse и Kafka"
|
||
${COMPOSE_BIN} up -d clickhouse kafka
|
||
wait_for_clickhouse
|
||
|
||
echo "Шаг 2: применение DDL"
|
||
bash "${SCRIPT_DIR}/apply_clickhouse_ddl.sh"
|
||
|
||
echo "Шаг 3: сборка образа генератора"
|
||
${COMPOSE_BIN} build generator
|
||
|
||
echo "Шаг 4: короткий backfill стартовой истории"
|
||
${COMPOSE_BIN} run --rm --no-deps \
|
||
-e GEN_RUN_MODE=backfill \
|
||
-e GEN_STATE_RESET=true \
|
||
-e GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
|
||
-e GEN_SEED="${GEN_SEED}" \
|
||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||
-e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||
-e GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
|
||
-e GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
|
||
-e GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
|
||
-e GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
|
||
-e GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
|
||
-e GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
|
||
-e GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
|
||
generator
|
||
|
||
echo "Шаг 5: первый batch STG -> ODS -> DDS -> DM"
|
||
sleep "${WAIT_STG_SECONDS}"
|
||
bash "${SCRIPT_DIR}/run_batch.sh"
|
||
stg_rows_before_live="$(stg_total_rows)"
|
||
|
||
echo "Шаг 6: live-продолжение, ждём новые STG-строки до ${LIVE_SECONDS} сек."
|
||
GEN_RUN_MODE=live \
|
||
GEN_STATE_RESET=false \
|
||
GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
|
||
GEN_SEED="${GEN_SEED}" \
|
||
GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||
GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||
GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
|
||
GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
|
||
GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
|
||
GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
|
||
GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
|
||
GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
|
||
GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
|
||
${COMPOSE_BIN} up -d --build generator
|
||
deadline=$((SECONDS + LIVE_SECONDS))
|
||
while true; do
|
||
stg_rows_after_live="$(stg_total_rows)"
|
||
if (( stg_rows_after_live > stg_rows_before_live )); then
|
||
echo "live_stg_rows_before=${stg_rows_before_live}"
|
||
echo "live_stg_rows_after=${stg_rows_after_live}"
|
||
break
|
||
fi
|
||
if (( SECONDS >= deadline )); then
|
||
echo "Ошибка: live-продолжение не записало новые STG-строки за ${LIVE_SECONDS} сек." >&2
|
||
exit 1
|
||
fi
|
||
sleep 1
|
||
done
|
||
cleanup_live_generator
|
||
|
||
echo "Шаг 7: второй batch после live"
|
||
sleep "${WAIT_STG_SECONDS}"
|
||
bash "${SCRIPT_DIR}/run_batch.sh"
|
||
|
||
echo "Шаг 8: проверка manifest/profile/live seam без Superset"
|
||
PROFILE="${PROFILE}" \
|
||
GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
|
||
GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES}" \
|
||
WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS}" \
|
||
CHECK_LIVE_SEAM=1 \
|
||
REQUIRE_SUPERSET=0 \
|
||
COMPOSE_BIN="${COMPOSE_BIN}" \
|
||
bash "${SCRIPT_DIR}/check_generated_analytics.sh"
|
||
|
||
echo ""
|
||
echo "Готово: runtime-проверка startup-history/live seam прошла."
|