diff --git a/Makefile b/Makefile index d07808a..966ae54 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: up down clean ddl data transform logs \ - generated-history-analytics generated-history-check \ + generated-history-analytics generated-history-check generated-history-runtime-check \ startup-history-export startup-history-import startup-history-check \ reload-monitoring recover-monitoring \ superset-init superset-dashboard superset-ui superset-restart \ @@ -46,7 +46,11 @@ generated-history-analytics: # Повторяемая проверка после прогона стартовой истории generated-history-check: - CHECK_LIVE_SEAM="$${CHECK_LIVE_SEAM:-auto}" COMPOSE_BIN="$(COMPOSE)" bash ./scripts/check_generated_analytics.sh + PROFILE="$${PROFILE:-$(PROFILE)}" GEN_LAUNCH_PROFILE="$${GEN_LAUNCH_PROFILE:-$${PROFILE:-$(PROFILE)}}" CHECK_LIVE_SEAM="$${CHECK_LIVE_SEAM:-1}" COMPOSE_BIN="$(COMPOSE)" bash ./scripts/check_generated_analytics.sh + +# Быстрый runtime gate issue 17: короткий daily-wave backfill + live seam без Superset UI +generated-history-runtime-check: + COMPOSE_BIN="$(COMPOSE)" bash ./scripts/run_generated_history_runtime_check.sh # Сгенерировать стартовую историю и сохранить портативный артефакт startup-history-export: diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 11ef263..5e0212e 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -188,13 +188,37 @@ CI короткая повторная проверка после уже гот make generated-history-check ``` -После live-продолжения из `T_end` эта же команда автоматически включает проверку -стыка. Для принудительной проверки: +По умолчанию команда требует live-строки после `T_end` и проверяет стык +backfill/live. Чистый `make generated-history-analytics` отключает эту часть +явно, потому что внутри него live-продолжение не запускается. Для ручной +backfill-only проверки используйте: + +```bash +CHECK_LIVE_SEAM=0 make generated-history-check +``` + +Для проверки другого live-окна: ```bash CHECK_LIVE_SEAM=1 GEN_LIVE_CHECK_MINUTES=10 make generated-history-check ``` +Для commit gate issue 17 есть короткий runtime-путь без полного `daily-wave` на +2 суток и без Superset UI: + +```bash +make generated-history-runtime-check +``` + +По умолчанию он берёт профиль `daily-wave`, но сжимает историю до `1h`, ждёт +новые STG-строки от live до 25 секунд, делает второй batch и проверяет стык с +`CHECK_LIVE_SEAM=1`. +Если машина медленная, можно увеличить только ожидания: + +```bash +LIVE_SECONDS=45 WAIT_STG_SECONDS=10 make generated-history-runtime-check +``` + По умолчанию команда использует быстрый профиль `ci`: 6 часов модельного времени. Историю на 2 суток с суточной волной можно получить одной командой. В live-продолжении `daily-wave` идёт с ×60 и тикает раз в секунду, поэтому diff --git a/docs/SUPERSET_DASHBOARD.md b/docs/SUPERSET_DASHBOARD.md index 725c818..4397705 100644 --- a/docs/SUPERSET_DASHBOARD.md +++ b/docs/SUPERSET_DASHBOARD.md @@ -25,6 +25,12 @@ make superset-init make superset-dashboard ``` +Повторный `make superset-dashboard` синхронизирует чарты этого учебного +дашборда с `CHARTS_CONFIG`: обновляет параметры, переименовывает старые имена и +может удалить лишний чарт-дубль. Удаление ограничено dashboard +`ecommerce-analytics`, поэтому одноимённые чарты менти в других dashboard не +трогаются. + ### 2. Доступ к UI Откройте в браузере: http://localhost:8088 diff --git a/generator/src/clickstream_generator/startup_history_artifact.py b/generator/src/clickstream_generator/startup_history_artifact.py index edf5fd9..db68a56 100644 --- a/generator/src/clickstream_generator/startup_history_artifact.py +++ b/generator/src/clickstream_generator/startup_history_artifact.py @@ -619,6 +619,39 @@ def _run_manifest_summary(args: argparse.Namespace) -> int: return 0 +def _print_manifest_summary(manifest: dict) -> None: + totals = manifest["totals"] + print( + "\t".join( + [ + str(totals["events"]), + str(totals["visits"]), + str(totals["users"]), + str(totals["min_event_timestamp"]), + str(totals["max_event_timestamp"]), + str(manifest["model_t0"]), + str(manifest["model_t_end"]), + str(manifest.get("launch_profile", "")), + ] + ) + ) + + +def _run_kafka_manifest_summary(args: argparse.Namespace) -> int: + config = Config() + manifest_manager = KafkaStartupHistoryManifest(config.kafka_bootstrap_servers) + try: + manifest = manifest_manager.load() + finally: + manifest_manager.close() + + if not manifest: + raise RuntimeError("startup history manifest not found in Kafka") + + _print_manifest_summary(manifest) + return 0 + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Startup history artifact tools") subparsers = parser.add_subparsers(dest="command", required=True) @@ -635,6 +668,9 @@ def main(argv: list[str] | None = None) -> int: summary_parser.add_argument("--artifact", required=True) summary_parser.set_defaults(func=_run_manifest_summary) + kafka_summary_parser = subparsers.add_parser("kafka-manifest-summary") + kafka_summary_parser.set_defaults(func=_run_kafka_manifest_summary) + args = parser.parse_args(argv) return args.func(args) diff --git a/scripts/check_generated_analytics.sh b/scripts/check_generated_analytics.sh index 00d82e9..f5ab409 100644 --- a/scripts/check_generated_analytics.sh +++ b/scripts/check_generated_analytics.sh @@ -12,8 +12,9 @@ CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-123456}" GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}" GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}" REQUIRE_SUPERSET="${REQUIRE_SUPERSET:-1}" -CHECK_LIVE_SEAM="${CHECK_LIVE_SEAM:-auto}" +CHECK_LIVE_SEAM="${CHECK_LIVE_SEAM:-1}" GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES:-10}" +WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS:-10}" fail() { echo "Ошибка: $*" >&2 @@ -50,11 +51,39 @@ CH_MODEL_T_END="$(clickhouse_datetime_literal "${GEN_MODEL_T_END}")" [[ "${GEN_LIVE_CHECK_MINUTES}" =~ ^[0-9]+$ ]] \ || fail "GEN_LIVE_CHECK_MINUTES должен быть целым числом минут" +[[ "${WAIT_LIVE_ROWS_SECONDS}" =~ ^[0-9]+$ ]] \ + || fail "WAIT_LIVE_ROWS_SECONDS должен быть целым числом секунд" case "${CHECK_LIVE_SEAM}" in 0|1|auto) ;; *) fail "CHECK_LIVE_SEAM должен быть 0, 1 или auto" ;; esac +echo "=== Проверка startup-history manifest ===" + +manifest_summary="$(${COMPOSE_BIN} run --rm --no-deps \ + generator python -m clickstream_generator.startup_history_artifact kafka-manifest-summary)" + +IFS=$'\t' read -r manifest_events manifest_visits manifest_users manifest_min_ts manifest_max_ts manifest_model_t0 manifest_model_t_end manifest_launch_profile <<< "${manifest_summary}" + +[[ "${manifest_events}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать startup-history manifest из Kafka" +[[ -n "${manifest_model_t0}" ]] || fail "startup-history manifest не содержит model_t0" +[[ -n "${manifest_model_t_end}" ]] || fail "startup-history manifest не содержит model_t_end" + +CH_MODEL_T0="$(clickhouse_datetime_literal "${manifest_model_t0}")" +CH_MODEL_T_END="$(clickhouse_datetime_literal "${manifest_model_t_end}")" + +expected_profile="${PROFILE:-${GEN_LAUNCH_PROFILE:-}}" +if [[ -n "${expected_profile}" && -n "${manifest_launch_profile}" && "${expected_profile}" != "${manifest_launch_profile}" ]]; then + fail "проверка запущена для профиля ${expected_profile}, но manifest от ${manifest_launch_profile}" +fi + +echo "manifest_events=${manifest_events}" +echo "manifest_visits=${manifest_visits}" +echo "manifest_users=${manifest_users}" +echo "manifest_model_t0=${manifest_model_t0}" +echo "manifest_model_t_end=${manifest_model_t_end}" +echo "manifest_launch_profile=${manifest_launch_profile}" + echo "=== Проверка ClickHouse: данные генерации в DM ===" stats_query=" @@ -271,7 +300,7 @@ echo "monotonic_ok=${contains_monotonic_ok}" echo "confirmation_share=${contains_confirmation_share}" should_check_live_seam="${CHECK_LIVE_SEAM}" -if [[ "${should_check_live_seam}" == "auto" ]]; then +if [[ "${should_check_live_seam}" != "0" ]]; then live_rows_query=" WITH toDateTime64('${CH_MODEL_T_END}', 6) AS t_end, @@ -280,12 +309,25 @@ SELECT count() FROM dds.event WHERE event_ts >= t_end AND event_ts < t_live_end FORMAT TabSeparated" - live_rows="$(ch_query "${live_rows_query}")" - [[ "${live_rows}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать live-строки DDS после GEN_MODEL_T_END" - if (( live_rows > 0 )); then - should_check_live_seam="1" - else - should_check_live_seam="0" + live_rows=0 + deadline=$((SECONDS + WAIT_LIVE_ROWS_SECONDS)) + while true; do + live_rows="$(ch_query "${live_rows_query}")" + [[ "${live_rows}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать live-строки DDS после GEN_MODEL_T_END" + if (( live_rows > 0 || SECONDS >= deadline )); then + break + fi + sleep 1 + done + if [[ "${should_check_live_seam}" == "1" ]] && (( live_rows == 0 )); then + fail "live-продолжение не записало строки в первые ${GEN_LIVE_CHECK_MINUTES} минут после manifest model_t_end" + fi + if [[ "${CHECK_LIVE_SEAM}" == "auto" ]]; then + if (( live_rows > 0 )); then + should_check_live_seam="1" + else + should_check_live_seam="0" + fi fi fi @@ -425,19 +467,35 @@ WITH ) SELECT (SELECT count() FROM crossing) AS crossing_visits, + ( + SELECT count() + FROM ods.device_by_click AS d + INNER JOIN crossing AS c ON c.click_id = d.click_id + WHERE d.click_id IS NOT NULL + ) AS ods_device_rows, + ( + SELECT count() + FROM ods.geo_by_click AS g + INNER JOIN crossing AS c ON c.click_id = g.click_id + WHERE g.click_id IS NOT NULL + ) AS ods_geo_rows, (SELECT count() FROM device_conflicts) AS ods_device_conflicts, (SELECT count() FROM geo_conflicts) AS ods_geo_conflicts FORMAT TabSeparated" ods_context="$(ch_query "${ods_context_query}")" - IFS=$'\t' read -r ods_crossing_visits ods_device_conflicts ods_geo_conflicts <<< "${ods_context}" + IFS=$'\t' read -r ods_crossing_visits ods_device_rows ods_geo_rows ods_device_conflicts ods_geo_conflicts <<< "${ods_context}" [[ "${ods_crossing_visits}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать ODS-проверку стыка" (( ods_crossing_visits == crossing_visits )) \ || fail "ODS и DDS нашли разное число переходящих визитов: ${ods_crossing_visits}/${crossing_visits}" + (( ods_device_rows > 0 )) || fail "ODS device пустой на стыке" + (( ods_geo_rows > 0 )) || fail "ODS geo пустой на стыке" (( ods_device_conflicts == 0 )) || fail "ODS device конфликтует на стыке: ${ods_device_conflicts}" (( ods_geo_conflicts == 0 )) || fail "ODS geo конфликтует на стыке: ${ods_geo_conflicts}" + echo "ods_device_rows=${ods_device_rows}" + echo "ods_geo_rows=${ods_geo_rows}" echo "ods_device_conflicts=${ods_device_conflicts}" echo "ods_geo_conflicts=${ods_geo_conflicts}" else diff --git a/scripts/run_generated_history_analytics.sh b/scripts/run_generated_history_analytics.sh index 6433b8c..586bcd9 100644 --- a/scripts/run_generated_history_analytics.sh +++ b/scripts/run_generated_history_analytics.sh @@ -105,6 +105,7 @@ ${COMPOSE_BIN} up -d --no-deps superset echo "Шаг 8: техническая проверка аналитического контура" GEN_MODEL_T0="${GEN_MODEL_T0}" \ GEN_MODEL_T_END="${GEN_MODEL_T_END}" \ +CHECK_LIVE_SEAM=0 \ COMPOSE_BIN="${COMPOSE_BIN}" \ bash "${SCRIPT_DIR}/check_generated_analytics.sh" diff --git a/scripts/run_generated_history_runtime_check.sh b/scripts/run_generated_history_runtime_check.sh new file mode 100644 index 0000000..9060277 --- /dev/null +++ b/scripts/run_generated_history_runtime_check.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# +# Быстрый runtime gate для issue 17: проверяет, что daily-wave доходит до +# manifest/check, а live-продолжение реально записывает стык backfill/live. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +COMPOSE_BIN="${COMPOSE_BIN:-docker compose}" +PROFILE="${PROFILE:-daily-wave}" +GEN_HISTORY_DURATION="${GEN_HISTORY_DURATION:-1h}" +WAIT_CLICKHOUSE_SECONDS="${WAIT_CLICKHOUSE_SECONDS:-60}" +WAIT_STG_SECONDS="${WAIT_STG_SECONDS:-5}" +LIVE_SECONDS="${LIVE_SECONDS:-25}" +GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES:-10}" +WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS:-2}" + +cd "${REPO_ROOT}" + +launch_env="$( + PYTHONPATH="${REPO_ROOT}/generator/src" \ + uv run python -m clickstream_generator.launch \ + backfill \ + --profile "${PROFILE}" \ + --duration "${GEN_HISTORY_DURATION}" +)" +eval "${launch_env}" + +wait_for_clickhouse() { + local deadline + deadline=$((SECONDS + WAIT_CLICKHOUSE_SECONDS)) + + until ${COMPOSE_BIN} exec -T clickhouse clickhouse-client \ + --user=default \ + --password=123456 \ + --query "SELECT 1" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo "Ошибка: ClickHouse не ответил за ${WAIT_CLICKHOUSE_SECONDS} сек." >&2 + exit 1 + fi + sleep 2 + done +} + +stg_total_rows() { + ${COMPOSE_BIN} exec -T clickhouse clickhouse-client \ + --user=default \ + --password=123456 \ + --query " +SELECT + (SELECT count() FROM stg.browser_raw) + + (SELECT count() FROM stg.location_raw) + + (SELECT count() FROM stg.device_raw) + + (SELECT count() FROM stg.geo_raw) +FORMAT TabSeparated" +} + +cleanup_live_generator() { + ${COMPOSE_BIN} stop generator >/dev/null 2>&1 || true +} +trap cleanup_live_generator EXIT + +echo "=== Быстрая runtime-проверка startup-history/live seam ===" +echo "GEN_LAUNCH_PROFILE=${GEN_LAUNCH_PROFILE}" +echo "GEN_HISTORY_DURATION=${GEN_HISTORY_DURATION}" +echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}" +echo "" + +echo "Шаг 0: очистка volumes ClickHouse/Kafka/state" +${COMPOSE_BIN} --profile live-generator down -v --remove-orphans + +echo "Шаг 1: запуск ClickHouse и Kafka" +${COMPOSE_BIN} up -d clickhouse kafka +wait_for_clickhouse + +echo "Шаг 2: применение DDL" +bash "${SCRIPT_DIR}/apply_clickhouse_ddl.sh" + +echo "Шаг 3: сборка образа генератора" +${COMPOSE_BIN} build generator + +echo "Шаг 4: короткий backfill стартовой истории" +${COMPOSE_BIN} run --rm --no-deps \ + -e GEN_RUN_MODE=backfill \ + -e GEN_STATE_RESET=true \ + -e GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \ + -e GEN_SEED="${GEN_SEED}" \ + -e GEN_MODEL_T0="${GEN_MODEL_T0}" \ + -e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \ + -e GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \ + -e GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \ + -e GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \ + -e GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \ + -e GEN_JITTER_PCT="${GEN_JITTER_PCT}" \ + -e GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \ + -e GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \ + generator + +echo "Шаг 5: первый batch STG -> ODS -> DDS -> DM" +sleep "${WAIT_STG_SECONDS}" +bash "${SCRIPT_DIR}/run_batch.sh" +stg_rows_before_live="$(stg_total_rows)" + +echo "Шаг 6: live-продолжение, ждём новые STG-строки до ${LIVE_SECONDS} сек." +GEN_RUN_MODE=live \ +GEN_STATE_RESET=false \ +GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \ +GEN_SEED="${GEN_SEED}" \ +GEN_MODEL_T0="${GEN_MODEL_T0}" \ +GEN_MODEL_T_END="${GEN_MODEL_T_END}" \ +GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \ +GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \ +GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \ +GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \ +GEN_JITTER_PCT="${GEN_JITTER_PCT}" \ +GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \ +GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \ +${COMPOSE_BIN} up -d --build generator +deadline=$((SECONDS + LIVE_SECONDS)) +while true; do + stg_rows_after_live="$(stg_total_rows)" + if (( stg_rows_after_live > stg_rows_before_live )); then + echo "live_stg_rows_before=${stg_rows_before_live}" + echo "live_stg_rows_after=${stg_rows_after_live}" + break + fi + if (( SECONDS >= deadline )); then + echo "Ошибка: live-продолжение не записало новые STG-строки за ${LIVE_SECONDS} сек." >&2 + exit 1 + fi + sleep 1 +done +cleanup_live_generator + +echo "Шаг 7: второй batch после live" +sleep "${WAIT_STG_SECONDS}" +bash "${SCRIPT_DIR}/run_batch.sh" + +echo "Шаг 8: проверка manifest/profile/live seam без Superset" +PROFILE="${PROFILE}" \ +GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \ +GEN_LIVE_CHECK_MINUTES="${GEN_LIVE_CHECK_MINUTES}" \ +WAIT_LIVE_ROWS_SECONDS="${WAIT_LIVE_ROWS_SECONDS}" \ +CHECK_LIVE_SEAM=1 \ +REQUIRE_SUPERSET=0 \ +COMPOSE_BIN="${COMPOSE_BIN}" \ +bash "${SCRIPT_DIR}/check_generated_analytics.sh" + +echo "" +echo "Готово: runtime-проверка startup-history/live seam прошла." diff --git a/superset/create_dashboard.py b/superset/create_dashboard.py index b4623ed..1a553cb 100644 --- a/superset/create_dashboard.py +++ b/superset/create_dashboard.py @@ -167,6 +167,8 @@ CHARTS_CONFIG = [ "row_limit": 15, "order_desc": True, "sort_series_type": "sum", + "x_axis_sort": "Events, pcs", + "x_axis_sort_asc": False, "orientation": "vertical", "color_scheme": "supersetColors", "show_legend": True, @@ -382,8 +384,14 @@ def sync_query_context(chart, params: dict, dataset_id: int) -> None: chart.query_context = json.dumps(query_context) -def choose_chart_to_sync(existing_charts: list, current_name: str): +def choose_chart_to_sync(existing_charts: list, current_name: str, dataset_id: int | None = None): """Выбирает один chart для синхронизации и отдаёт лишние дубли на удаление.""" + if dataset_id is not None: + existing_charts = [ + chart + for chart in existing_charts + if getattr(chart, "datasource_id", None) == dataset_id + ] if not existing_charts: return None, [] @@ -395,6 +403,16 @@ def choose_chart_to_sync(existing_charts: list, current_name: str): return selected, duplicates +def dashboard_owned_charts(existing_charts: list, dashboard_slug: str) -> list: + """Оставляет только charts, уже привязанные к целевому dashboard.""" + owned = [] + for chart in existing_charts: + dashboards = getattr(chart, "dashboards", []) or [] + if any(getattr(dashboard, "slug", None) == dashboard_slug for dashboard in dashboards): + owned.append(chart) + return owned + + def build_dashboard_metadata(filter_dataset_id: int | None) -> str: """Формирует json_metadata с валидными datasetId для native filters.""" native_filters = [] @@ -470,6 +488,9 @@ def main() -> bool: created_charts = [] datasets_by_name = {} + existing_dashboard = db.session.query(Dashboard).filter_by( + slug=DASHBOARD_CONFIG["slug"] + ).first() # Создаём чарты for chart_config in CHARTS_CONFIG: @@ -503,11 +524,18 @@ def main() -> bool: chart_names = [chart_config["slice_name"]] chart_names.extend(chart_config.get("previous_slice_names", [])) existing_charts = db.session.query(Slice).filter( - Slice.slice_name.in_(chart_names) + Slice.slice_name.in_(chart_names), + Slice.datasource_id == dataset.id, ).order_by(Slice.id.asc()).all() + if existing_dashboard: + existing_charts = dashboard_owned_charts( + existing_charts, + DASHBOARD_CONFIG["slug"], + ) existing, duplicate_charts = choose_chart_to_sync( existing_charts, chart_config["slice_name"], + dataset_id=dataset.id, ) if existing: @@ -562,8 +590,19 @@ def main() -> bool: logger.info(f"Created/Found {len(created_charts)} charts") current_chart_names = {chart_config["slice_name"] for chart_config in CHARTS_CONFIG} + target_dataset_ids = set(datasets_by_name.values()) for obsolete_name in sorted(OBSOLETE_CHART_NAMES - current_chart_names): - obsolete_charts = db.session.query(Slice).filter_by(slice_name=obsolete_name).all() + if not existing_dashboard: + obsolete_charts = [] + else: + obsolete_charts = db.session.query(Slice).filter( + Slice.slice_name == obsolete_name, + Slice.datasource_id.in_(target_dataset_ids), + ).all() + obsolete_charts = dashboard_owned_charts( + obsolete_charts, + DASHBOARD_CONFIG["slug"], + ) for obsolete in obsolete_charts: db.session.delete(obsolete) logger.info("Deleted obsolete chart: %s (ID: %s)", obsolete_name, obsolete.id) @@ -651,9 +690,7 @@ def main() -> bool: if created_charts: try: # Проверяем, существует ли дашборд - existing = db.session.query(Dashboard).filter_by( - slug=DASHBOARD_CONFIG["slug"] - ).first() + existing = existing_dashboard if existing: existing.description = DASHBOARD_CONFIG["description"] diff --git a/superset/dashboards/ecommerce_analytics.zip.json b/superset/dashboards/ecommerce_analytics.zip.json index 7814a13..a92616c 100644 --- a/superset/dashboards/ecommerce_analytics.zip.json +++ b/superset/dashboards/ecommerce_analytics.zip.json @@ -90,7 +90,7 @@ "viz_type": "echarts_timeseries_bar", "datasource_type": "table", "datasource_name": "dm.v_events_enriched", - "params": "{\"x_axis\": \"geo_country\", \"metrics\": [{\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(*)\", \"label\": \"Events, pcs\"}], \"row_limit\": 15, \"order_desc\": true, \"sort_series_type\": \"sum\", \"orientation\": \"vertical\", \"color_scheme\": \"supersetColors\", \"show_legend\": true, \"legendOrientation\": \"top\", \"legendType\": \"scroll\", \"rich_tooltip\": true, \"tooltipTimeFormat\": \"smart_date\", \"x_axis_title\": \"Country\", \"x_axis_title_margin\": 15, \"truncateXAxis\": true, \"y_axis_title\": \"Events, pcs\", \"y_axis_title_margin\": 15, \"y_axis_title_position\": \"Left\", \"y_axis_format\": \",d\", \"time_range\": \"No filter\", \"datasource\": \"1__table\", \"viz_type\": \"echarts_timeseries_bar\"}", + "params": "{\"x_axis\": \"geo_country\", \"metrics\": [{\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(*)\", \"label\": \"Events, pcs\"}], \"row_limit\": 15, \"order_desc\": true, \"sort_series_type\": \"sum\", \"x_axis_sort\": \"Events, pcs\", \"x_axis_sort_asc\": false, \"orientation\": \"vertical\", \"color_scheme\": \"supersetColors\", \"show_legend\": true, \"legendOrientation\": \"top\", \"legendType\": \"scroll\", \"rich_tooltip\": true, \"tooltipTimeFormat\": \"smart_date\", \"x_axis_title\": \"Country\", \"x_axis_title_margin\": 15, \"truncateXAxis\": true, \"y_axis_title\": \"Events, pcs\", \"y_axis_title_margin\": 15, \"y_axis_title_position\": \"Left\", \"y_axis_format\": \",d\", \"time_range\": \"No filter\", \"datasource\": \"1__table\", \"viz_type\": \"echarts_timeseries_bar\"}", "description": "Chart created automatically for v_events_enriched" } }, diff --git a/tests/test_generated_analytics_check_contract.py b/tests/test_generated_analytics_check_contract.py new file mode 100644 index 0000000..85d7ca0 --- /dev/null +++ b/tests/test_generated_analytics_check_contract.py @@ -0,0 +1,72 @@ +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_generated_history_check_uses_actual_manifest_boundary_and_profile(): + """Проверка берёт стык и профиль из manifest фактического прогона.""" + script = (REPO_ROOT / "scripts" / "check_generated_analytics.sh").read_text( + encoding="utf-8" + ) + makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") + + assert "kafka-manifest-summary" in script + assert "manifest_model_t_end" in script + assert "CH_MODEL_T_END=\"$(clickhouse_datetime_literal \"${manifest_model_t_end}\")\"" in script + assert "GEN_LAUNCH_PROFILE" in makefile + assert "PROFILE" in makefile + + +def test_generated_history_check_does_not_skip_required_live_seam(): + """Обычная проверка требует реальные live-строки после стыка.""" + script = (REPO_ROOT / "scripts" / "check_generated_analytics.sh").read_text( + encoding="utf-8" + ) + makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") + + assert 'CHECK_LIVE_SEAM="${CHECK_LIVE_SEAM:-1}"' in script + assert 'if [[ "${should_check_live_seam}" == "1" ]] && (( live_rows == 0 )); then' in script + assert "live-продолжение не записало строки" in script + assert 'CHECK_LIVE_SEAM="$${CHECK_LIVE_SEAM:-1}"' in makefile + + +def test_clean_generated_history_run_explicitly_skips_live_seam(): + """Чистый backfill без live-продолжения отключает проверку стыка явно.""" + script = (REPO_ROOT / "scripts" / "run_generated_history_analytics.sh").read_text( + encoding="utf-8" + ) + + assert "CHECK_LIVE_SEAM=0" in script + assert script.index("CHECK_LIVE_SEAM=0") < script.index('bash "${SCRIPT_DIR}/check_generated_analytics.sh"') + + +def test_runtime_gate_has_bounded_daily_wave_live_seam_path(): + """Runtime gate issue 17 не использует полный daily-wave на 2 суток.""" + makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") + script = (REPO_ROOT / "scripts" / "run_generated_history_runtime_check.sh").read_text( + encoding="utf-8" + ) + + assert "generated-history-runtime-check" in makefile + assert 'PROFILE="${PROFILE:-daily-wave}"' in script + assert 'GEN_HISTORY_DURATION="${GEN_HISTORY_DURATION:-1h}"' in script + assert 'LIVE_SECONDS="${LIVE_SECONDS:-25}"' in script + assert "stg_rows_after_live > stg_rows_before_live" in script + assert "GEN_RUN_MODE=live" in script + assert "GEN_STATE_RESET=false" in script + assert "up -d --build generator" in script + assert "CHECK_LIVE_SEAM=1" in script + assert "REQUIRE_SUPERSET=0" in script + + +def test_generated_history_check_rejects_empty_ods_context(): + """ODS-блок стыка падает, если в ODS нет строк для переходящих визитов.""" + script = (REPO_ROOT / "scripts" / "check_generated_analytics.sh").read_text( + encoding="utf-8" + ) + + assert "ods_device_rows" in script + assert "ods_geo_rows" in script + assert "ODS device пустой на стыке" in script + assert "ODS geo пустой на стыке" in script diff --git a/tests/test_superset_dashboard_config.py b/tests/test_superset_dashboard_config.py index d494c69..e39fe43 100644 --- a/tests/test_superset_dashboard_config.py +++ b/tests/test_superset_dashboard_config.py @@ -28,10 +28,28 @@ def load_exported_dashboard(): return json.loads(export_path.read_text()) +def expected_export_params(chart, actual_params): + return { + **chart["params"], + "datasource": actual_params["datasource"], + "viz_type": chart["viz_type"], + } + + +class FakeDashboard: + def __init__(self, slug): + self.slug = slug + + class FakeChart: - def __init__(self, chart_id, slice_name): + def __init__(self, chart_id, slice_name, datasource_id=1, dashboard_slugs=None): self.id = chart_id self.slice_name = slice_name + self.datasource_id = datasource_id + self.dashboards = [ + FakeDashboard(slug) + for slug in (dashboard_slugs or ["ecommerce-analytics"]) + ] def test_geo_chart_uses_readable_top_countries_bar_config(): @@ -62,15 +80,20 @@ def test_geo_chart_uses_readable_top_countries_bar_config(): def test_exported_dashboard_uses_same_readable_geo_chart(): + module = load_dashboard_module() exported_dashboard = load_exported_dashboard() + expected_chart = chart_config(module, "🌍 Top Countries by Events") geo_chart = exported_chart(exported_dashboard, "🌍 Top Countries by Events") assert geo_chart["viz_type"] == "echarts_timeseries_bar" params = json.loads(geo_chart["params"]) + assert params == expected_export_params(expected_chart, params) assert params["viz_type"] == "echarts_timeseries_bar" assert params["x_axis"] == "geo_country" assert params["metrics"][0]["label"] == "Events, pcs" + assert params["x_axis_sort"] == "Events, pcs" + assert params["x_axis_sort_asc"] is False assert params["show_legend"] is True assert params["rich_tooltip"] is True assert params["y_axis_title"] == "Events, pcs" @@ -79,6 +102,24 @@ def test_exported_dashboard_uses_same_readable_geo_chart(): assert position_json["CHART-7"]["meta"]["sliceName"] == "🌍 Top Countries by Events" +def test_exported_dashboard_chart_params_match_current_config(): + module = load_dashboard_module() + exported_dashboard = load_exported_dashboard() + + expected_charts = { + chart["slice_name"]: chart + for chart in module.CHARTS_CONFIG + } + + for exported in exported_dashboard["charts"]: + actual_chart = exported["__Slice__"] + actual_params = json.loads(actual_chart["params"]) + expected_chart = expected_charts[actual_chart["slice_name"]] + + assert actual_chart["viz_type"] == expected_chart["viz_type"] + assert actual_params == expected_export_params(expected_chart, actual_params) + + def test_exported_dashboard_chart_names_match_current_config(): module = load_dashboard_module() exported_dashboard = load_exported_dashboard() @@ -118,13 +159,59 @@ def test_exported_dashboard_layout_matches_current_rows(): def test_choose_chart_to_sync_prefers_current_name_and_marks_old_name_duplicate(): module = load_dashboard_module() - old_chart = FakeChart(7, "🌍 Geography Map") - current_chart = FakeChart(11, "🌍 Top Countries by Events") + old_chart = FakeChart(7, "🌍 Geography Map", datasource_id=42) + current_chart = FakeChart(11, "🌍 Top Countries by Events", datasource_id=42) selected, duplicates = module.choose_chart_to_sync( [old_chart, current_chart], "🌍 Top Countries by Events", + dataset_id=42, ) assert selected is current_chart assert duplicates == [old_chart] + + +def test_choose_chart_to_sync_ignores_same_name_from_other_dataset(): + module = load_dashboard_module() + other_dataset_chart = FakeChart(7, "🌍 Top Countries by Events", datasource_id=7) + target_dataset_chart = FakeChart(11, "🌍 Geography Map", datasource_id=42) + + selected, duplicates = module.choose_chart_to_sync( + [other_dataset_chart, target_dataset_chart], + "🌍 Top Countries by Events", + dataset_id=42, + ) + + assert selected is target_dataset_chart + assert duplicates == [] + + +def test_dashboard_sync_ignores_same_name_same_dataset_from_other_dashboard(): + module = load_dashboard_module() + other_dashboard_chart = FakeChart( + 7, + "🌍 Top Countries by Events", + datasource_id=42, + dashboard_slugs=["mentee-dashboard"], + ) + target_dashboard_chart = FakeChart( + 11, + "🌍 Geography Map", + datasource_id=42, + dashboard_slugs=["ecommerce-analytics"], + ) + + owned_charts = module.dashboard_owned_charts( + [other_dashboard_chart, target_dashboard_chart], + dashboard_slug="ecommerce-analytics", + ) + selected, duplicates = module.choose_chart_to_sync( + owned_charts, + "🌍 Top Countries by Events", + dataset_id=42, + ) + + assert owned_charts == [target_dashboard_chart] + assert selected is target_dashboard_chart + assert duplicates == []