ddadminandClaude Fable 5
f5dee26eda
- Зачем:
- режим кормления стенда порциями: менти триггерит «следующий день»,
видит полный цикл DWH за один шаг (задача 13, вариант 2 — генерация
от слепка T_end).
- Что:
- новый ограниченный режим next-day: восстановление мира из state,
генерация ровно [T_end, T_end+24h), публикация данные -> state ->
манифест (манифест — точка фиксации, автоотката нет).
- операция next-day в DAG generator_control: своя предпроверка границы
вместо clean-guard, идемпотентность через параметр expected_t_end.
- цепочка границ — накопительное поле boundaries в манифесте, старый
формат читается как [T0, T_end]; импорт не изменён.
- новая проверка цепочки (make generated-history-chain-check): непарные
счётчики и однородность по каждой границе, явный статус нулевого
стыка, хвост за границей по всем четырём топикам, литералы в UTC
с микросекундами.
- документация OPERATIONS.md: глагол, предпроверка, восстановление
после сбоя, ограничение retention; в задаче 13 — решения двух слепых
ревью постановки и кода с аргументами отклонений.
- Проверка:
- make test: 204 теста генератора + 31 контракт корня, зелёные.
- make generated-history-chain-check: зелёный, 2 внутренние границы,
непарные счётчики нулевые; учебный цикл: DM 322 -> 10026 -> 19196
за два next-day подряд.
- make generated-history-runtime-check (регрессия задачи 20): зелёный.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
241 lines
9.8 KiB
Python
241 lines
9.8 KiB
Python
"""
|
||
Контракт DAG generator_control без запуска Airflow.
|
||
"""
|
||
|
||
import ast
|
||
from pathlib import Path
|
||
|
||
from clickstream_generator.launch import PROFILES
|
||
|
||
|
||
DAG_PATH = Path(__file__).parents[2] / "airflow" / "dags" / "generator_control_dag.py"
|
||
REPO_ROOT = Path(__file__).parents[2]
|
||
|
||
|
||
def _tree():
|
||
return ast.parse(DAG_PATH.read_text(encoding="utf-8"))
|
||
|
||
|
||
def test_dag_file_exists_and_uses_dynamic_profiles():
|
||
"""DAG берёт варианты профилей из PROFILES, а не из ручного списка."""
|
||
text = DAG_PATH.read_text(encoding="utf-8")
|
||
|
||
assert "dag_id=\"generator_control\"" in text
|
||
assert "sorted(PROFILES)" in text
|
||
for profile in PROFILES:
|
||
assert profile not in {"hardcoded-profile"}
|
||
|
||
|
||
def test_trigger_form_has_expected_param_enums():
|
||
"""Форма запуска ограничивает операции и профили."""
|
||
text = DAG_PATH.read_text(encoding="utf-8")
|
||
|
||
assert 'enum=["backfill", "import", "next-day", "check"]' in text
|
||
assert "enum=sorted(PROFILES)" in text
|
||
assert '"duration": Param(' in text
|
||
assert '"artifact_path": Param(' in text
|
||
assert '"expected_t_end": Param(' in text
|
||
|
||
|
||
def test_dag_branches_and_waits_for_etl_completion():
|
||
"""Backfill/import/next-day запускают ETL и ждут завершения перед check."""
|
||
text = DAG_PATH.read_text(encoding="utf-8")
|
||
|
||
assert "BranchPythonOperator" in text
|
||
assert "TriggerDagRunOperator" in text
|
||
assert "trigger_dag_id=ETL_DAG_ID" in text
|
||
assert "wait_for_completion=True" in text
|
||
assert 'allowed_states=["success"]' in text
|
||
assert 'failed_states=["failed"]' in text
|
||
|
||
|
||
def test_next_day_has_own_boundary_precheck_and_serial_execution():
|
||
"""Next-day не использует clean-guard и не допускает параллельных запусков."""
|
||
text = DAG_PATH.read_text(encoding="utf-8")
|
||
|
||
assert "schedule=None" in text
|
||
assert "max_active_runs=1" in text
|
||
assert 'return "check_etl_not_paused_before_next_day"' in text
|
||
assert "assert_next_day_snapshot" in text
|
||
assert "assert_expected_t_end" in text
|
||
next_day_precheck = text.split("def precheck_next_day", maxsplit=1)[1].split(
|
||
"\ndef ", maxsplit=1
|
||
)[0]
|
||
assert "assert_stand_clean" not in next_day_precheck
|
||
assert "assert_live_generator_not_running" in next_day_precheck
|
||
assert "run_next_day" in text
|
||
|
||
|
||
def test_generator_control_prechecks_etl_dag_not_paused_before_waiting():
|
||
"""Пульт проверяет паузу etl_pipeline до долгого ожидания."""
|
||
text = DAG_PATH.read_text(encoding="utf-8")
|
||
|
||
assert "assert_target_dag_not_paused" in text
|
||
assert 'ETL_DAG_ID = "etl_pipeline"' in text
|
||
assert 'op_kwargs={"dag_id": ETL_DAG_ID}' in text
|
||
assert "session.query(DagModel)" in text
|
||
assert "DagModel.dag_id == dag_id" in text
|
||
assert "target_dag_trigger_error" in text
|
||
assert "Airflow 2.10.5" in text
|
||
assert "fail_when_dag_is_paused" in text
|
||
assert 'return "check_etl_not_paused_before_backfill"' in text
|
||
assert 'return "check_etl_not_paused_before_import"' in text
|
||
assert 'return "check_etl_not_paused_before_next_day"' in text
|
||
assert "check_etl_not_paused_before_backfill >> precheck_backfill_task" in text
|
||
assert "check_etl_not_paused_before_import >> precheck_import_task" in text
|
||
assert "check_etl_not_paused_before_next_day >> precheck_next_day_task" in text
|
||
assert text.index("check_etl_not_paused_before_backfill >> precheck_backfill_task") < text.index(
|
||
"precheck_backfill_task >> backfill_task"
|
||
)
|
||
assert text.index("check_etl_not_paused_before_import >> precheck_import_task") < text.index(
|
||
"precheck_import_task >> import_task"
|
||
)
|
||
|
||
|
||
def test_make_up_rebuilds_airflow_images_after_repo_update():
|
||
"""make up пересобирает Airflow, но не запускает Superset до DM."""
|
||
text = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
|
||
|
||
up_block = text.split("\nup:\n", maxsplit=1)[1].split(
|
||
"\n\n# Остановить",
|
||
maxsplit=1,
|
||
)[0]
|
||
assert "$(COMPOSE) up -d --build" in up_block
|
||
assert "airflow-webserver" in up_block
|
||
assert "airflow-scheduler" in up_block
|
||
assert "superset" not in up_block
|
||
assert "superset-init" not in up_block
|
||
|
||
|
||
def test_readme_quick_start_lists_prerequisites_and_runs_ddl_before_generator_control():
|
||
"""Быстрый старт называет инструменты и DDL до generator_control."""
|
||
text = (REPO_ROOT / "README.md").read_text(encoding="utf-8")
|
||
|
||
assert "uv" in text
|
||
assert "Docker" in text
|
||
assert "docker compose" in text
|
||
assert "make up\nmake ddl" in text
|
||
assert text.index("make ddl") < text.index("generator_control")
|
||
|
||
|
||
def test_course_readme_lists_uv_before_first_command():
|
||
"""Курс называет uv до первой команды."""
|
||
text = (REPO_ROOT / "docs" / "course" / "README.md").read_text(encoding="utf-8")
|
||
|
||
assert "uv" in text
|
||
assert text.index("uv") < text.index("make generated-history-analytics")
|
||
|
||
|
||
def test_startup_history_runbook_warns_about_daily_wave_idle_gap():
|
||
"""Runbook предупреждает о дыре модельного времени при простое daily-wave."""
|
||
text = (REPO_ROOT / "docs" / "runbooks" / "startup-history.md").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
|
||
assert "daily-wave" in text
|
||
assert "прост" in text
|
||
assert "дыр" in text
|
||
assert "make generator-reset" in text
|
||
assert "startup-history-import" in text
|
||
|
||
|
||
def test_no_docker_or_continue_operation_in_dag():
|
||
"""Пульт не управляет Docker и не содержит операцию continue."""
|
||
text = DAG_PATH.read_text(encoding="utf-8")
|
||
|
||
assert "docker" not in text.lower()
|
||
assert '"continue"' not in text
|
||
|
||
|
||
def test_compose_mounts_generator_code_without_socket_and_gates_live_service():
|
||
"""Airflow видит код генератора, но не получает Docker socket."""
|
||
text = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||
|
||
assert "PYTHONPATH: /opt/airflow/generator_src" in text
|
||
assert "./generator/src:/opt/airflow/generator_src:ro" in text
|
||
assert "/var/run/docker.sock" not in text
|
||
assert "profiles:\n - live-generator" in text
|
||
|
||
|
||
def test_down_and_clean_include_live_generator_profile():
|
||
"""Остановка стенда удаляет профильный live-генератор."""
|
||
text = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
|
||
|
||
assert "down:\n\t$(COMPOSE) --profile live-generator down" in text
|
||
assert "clean:\n\t$(COMPOSE) --profile live-generator down -v --remove-orphans" in text
|
||
|
||
|
||
def test_console_generator_paths_precheck_clean_stand_before_writes():
|
||
"""Backfill/import с хоста проверяют чистый стенд до записи."""
|
||
run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
import_artifact = (
|
||
REPO_ROOT / "scripts" / "import_startup_history_artifact.sh"
|
||
).read_text(encoding="utf-8")
|
||
|
||
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in run_generator
|
||
assert run_generator.index("assert_stand_clean.sh") < run_generator.index(
|
||
"run --rm --no-deps"
|
||
)
|
||
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in import_artifact
|
||
assert import_artifact.index("assert_stand_clean.sh") < import_artifact.index(
|
||
"startup_history_artifact import"
|
||
)
|
||
|
||
|
||
def test_console_continue_checks_state_or_clean_stand_before_live_start():
|
||
"""Continue не стартует новый live-мир на грязном стенде без state."""
|
||
run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
|
||
assert "elif [[ \"${VERB}\" == \"continue\" ]]" in run_generator
|
||
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" continue" in run_generator
|
||
assert run_generator.index("assert_stand_clean.sh\" continue") < run_generator.index(
|
||
"up -d --build generator"
|
||
)
|
||
|
||
|
||
def test_console_reset_checks_clean_stand_before_live_start():
|
||
"""Reset не пишет новый мир поверх старых данных."""
|
||
run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
reset_branch = run_generator.split("else\n", maxsplit=1)[1]
|
||
|
||
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in reset_branch
|
||
assert reset_branch.index("assert_stand_clean.sh\" clean") < reset_branch.index(
|
||
"up -d --build generator"
|
||
)
|
||
|
||
|
||
def test_clean_start_paths_include_live_generator_profile():
|
||
"""Все чистые сбросы видят профильный live-генератор."""
|
||
generated_history = (
|
||
REPO_ROOT / "scripts" / "run_generated_history_analytics.sh"
|
||
).read_text(encoding="utf-8")
|
||
|
||
assert "--profile live-generator down -v --remove-orphans" in generated_history
|
||
|
||
|
||
def test_host_clean_precheck_checks_live_before_kafka_and_stg():
|
||
"""Host backfill/import сначала проверяет, что live-генератор не запущен."""
|
||
text = (REPO_ROOT / "scripts" / "assert_stand_clean.sh").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
|
||
assert "--mode live" in text
|
||
assert text.index("--mode live") < text.index("clickhouse-client")
|
||
assert text.index("clickhouse-client") < text.index("--mode \"${CHECK_MODE}\"")
|
||
|
||
|
||
def test_airflow_and_generator_kafka_dependency_versions_match():
|
||
"""Airflow и генератор используют одну версию kafka-python."""
|
||
airflow_req = (REPO_ROOT / "airflow" / "requirements.txt").read_text(encoding="utf-8")
|
||
generator_req = (REPO_ROOT / "generator" / "requirements.txt").read_text(encoding="utf-8")
|
||
|
||
assert "kafka-python==2.0.6" in airflow_req
|
||
assert "kafka-python==2.0.6" in generator_req
|
||
assert "prometheus-client==0.21.1" in airflow_req
|