feat(airflow): пульт стал world_init, добавлен DAG world_next_day (#4)

- Зачем: - список DAG'ов должен читаться лесенкой ddl_init → world_init → world_next_day, а путь менти — проходиться пустыми формами (issue #4, спека редизайна пути менти, решения 2–3). - Что: - generator_control переименован в world_init, дефолт операции — import; next-day ушёл из выпадашки в отдельный DAG; - новый беспараметрный world_next_day: расписание */30 * * * *, создаётся на паузе, catchup=False, max_active_runs=1; общие задачи вынесены в airflow/dags/utils/startup_history_tasks.py; - доки и контрактные тесты обновлены синхронно; быстрый старт README — без make ddl, схему создаёт DAG ddl_init. - Проверка: - make test (210 + 31) и make lint зелёные; - живая приёмка на чистом стенде: world_init пустой формой импортировал эталонный мир за 217 с (3 дня, 280 437 событий), world_next_day после снятия с паузы добавляет ровно один день за прогон, дашборд Superset собирается. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-22 21:45:02 +03:00
parent 76ff669cf7
commit ac7a973504
12 changed files with 268 additions and 192 deletions
@@ -0,0 +1,321 @@
+"""Контракты DAG world_init и world_next_day без запуска Airflow."""
+
+import ast
+from pathlib import Path
+
+from clickstream_generator.launch import PROFILES
+
+
+DAG_PATH = Path(__file__).parents[2] / "airflow" / "dags" / "world_init_dag.py"
+NEXT_DAY_DAG_PATH = Path(__file__).parents[2] / "airflow" / "dags" / "world_next_day_dag.py"
+REPO_ROOT = Path(__file__).parents[2]
+
+
+def _tree():
+    return ast.parse(DAG_PATH.read_text(encoding="utf-8"))
+
+
+def _declared_param_types() -> dict[str, set[str]]:
+    param_types = {}
+    for node in ast.walk(_tree()):
+        if not isinstance(node, ast.Dict):
+            continue
+        for key, value in zip(node.keys, node.values, strict=True):
+            if not (
+                isinstance(key, ast.Constant)
+                and isinstance(key.value, str)
+                and isinstance(value, ast.Call)
+                and isinstance(value.func, ast.Name)
+                and value.func.id == "Param"
+            ):
+                continue
+            type_keyword = next(
+                keyword.value for keyword in value.keywords if keyword.arg == "type"
+            )
+            if isinstance(type_keyword, ast.List):
+                param_types[key.value] = {
+                    item.value for item in type_keyword.elts if isinstance(item, ast.Constant)
+                }
+            else:
+                param_types[key.value] = {type_keyword.value}
+    return param_types
+
+
+def _declared_param_defaults() -> dict[str, object]:
+    param_defaults = {}
+    for node in ast.walk(_tree()):
+        if not isinstance(node, ast.Dict):
+            continue
+        for key, value in zip(node.keys, node.values, strict=True):
+            if not (
+                isinstance(key, ast.Constant)
+                and isinstance(key.value, str)
+                and isinstance(value, ast.Call)
+                and isinstance(value.func, ast.Name)
+                and value.func.id == "Param"
+                and value.args
+            ):
+                continue
+            param_defaults[key.value] = ast.literal_eval(value.args[0])
+    return param_defaults
+
+
+def test_dag_file_exists_and_uses_dynamic_profiles():
+    """DAG берёт варианты профилей из PROFILES, а не из ручного списка."""
+    text = DAG_PATH.read_text(encoding="utf-8")
+
+    assert "dag_id=\"world_init\"" in text
+    assert "sorted(PROFILES)" in text
+    for profile in PROFILES:
+        assert profile not in {"hardcoded-profile"}
+
+
+def test_trigger_form_has_expected_param_enums():
+    """Форма запуска ограничивает операции и профили."""
+    text = DAG_PATH.read_text(encoding="utf-8")
+    param_defaults = _declared_param_defaults()
+
+    assert 'enum=["backfill", "import", "check"]' in text
+    assert param_defaults["operation"] == "import"
+    assert "enum=sorted(PROFILES)" in text
+    assert param_defaults["profile"] == "daily-wave"
+    assert '"duration": Param(' in text
+    assert '"artifact_path": Param(' in text
+    assert '"expected_t_end": Param(' not in text
+
+
+def test_trigger_form_marks_only_optional_params_as_nullable():
+    """Форма разрешает оставить необязательные поля пустыми."""
+    param_types = _declared_param_types()
+
+    for name in {
+        "duration",
+        "seed",
+        "model_time_speed",
+        "artifact_path",
+    }:
+        assert "null" in param_types[name]
+    for name in {"operation", "profile"}:
+        assert "null" not in param_types[name]
+
+
+def test_dag_branches_and_waits_for_etl_completion():
+    """Backfill/import запускают ETL и ждут завершения перед check."""
+    text = DAG_PATH.read_text(encoding="utf-8")
+
+    assert "BranchPythonOperator" in text
+    assert "TriggerDagRunOperator" in text
+    assert "trigger_dag_id=ETL_DAG_ID" in text
+    assert "wait_for_completion=True" in text
+    assert 'allowed_states=["success"]' in text
+    assert 'failed_states=["failed"]' in text
+
+
+def test_world_next_day_is_parameterless_paused_half_hour_dag():
+    """DAG следующего дня запускается пустой формой каждые полчаса."""
+    text = NEXT_DAY_DAG_PATH.read_text(encoding="utf-8")
+
+    assert 'dag_id="world_next_day"' in text
+    assert 'schedule="*/30 * * * *"' in text
+    assert "catchup=False" in text
+    assert "is_paused_upon_creation=True" in text
+    assert "max_active_runs=1" in text
+    assert "params=" not in text
+    assert "precheck_next_day" in text
+    assert "run_next_day_task" in text
+    assert "TriggerDagRunOperator" in text
+    assert "wait_for_completion=True" in text
+    assert "check_manifest_task" in text
+    assert "check_etl_not_paused >> precheck" in text
+
+
+def test_world_dags_precheck_etl_dag_not_paused_before_waiting():
+    """Пульт проверяет паузу etl_pipeline до долгого ожидания."""
+    text = DAG_PATH.read_text(encoding="utf-8")
+    shared_tasks = (
+        REPO_ROOT / "airflow" / "dags" / "utils" / "startup_history_tasks.py"
+    ).read_text(encoding="utf-8")
+
+    assert "assert_target_dag_not_paused" in text
+    assert 'ETL_DAG_ID = "etl_pipeline"' in text
+    assert 'op_kwargs={"dag_id": ETL_DAG_ID}' in text
+    assert "session.query(DagModel)" in shared_tasks
+    assert "DagModel.dag_id == dag_id" in shared_tasks
+    assert "target_dag_trigger_error" in shared_tasks
+    assert "Airflow 2.10.5" in shared_tasks
+    assert "fail_when_dag_is_paused" in shared_tasks
+    assert 'return "check_etl_not_paused_before_backfill"' in text
+    assert 'return "check_etl_not_paused_before_import"' in text
+    assert "check_etl_not_paused_before_backfill >> precheck_backfill_task" in text
+    assert "check_etl_not_paused_before_import >> precheck_import_task" in text
+    assert text.index("check_etl_not_paused_before_backfill >> precheck_backfill_task") < text.index(
+        "precheck_backfill_task >> backfill_task"
+    )
+    assert text.index("check_etl_not_paused_before_import >> precheck_import_task") < text.index(
+        "precheck_import_task >> import_task"
+    )
+
+
+def test_make_up_rebuilds_airflow_images_after_repo_update():
+    """make up пересобирает Airflow, но не запускает Superset до DM."""
+    text = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
+
+    up_block = text.split("\nup:\n", maxsplit=1)[1].split(
+        "\n\n# Остановить",
+        maxsplit=1,
+    )[0]
+    assert "$(COMPOSE) up -d --build" in up_block
+    assert "airflow-webserver" in up_block
+    assert "airflow-scheduler" in up_block
+    assert "superset" not in up_block
+    assert "superset-init" not in up_block
+
+
+def test_readme_quick_start_creates_schema_via_ddl_init_before_world_init():
+    """Быстрый старт: терминал — только make up, схему создаёт ddl_init до world_init."""
+    text = (REPO_ROOT / "README.md").read_text(encoding="utf-8")
+
+    assert "uv" in text
+    assert "Docker" in text
+    assert "docker compose" in text
+    assert "make ddl" not in text
+    quick_start = text.split("## Быстрый старт", maxsplit=1)[1]
+    assert quick_start.index("ddl_init") < quick_start.index("world_init")
+
+
+def test_course_readme_lists_uv_before_first_command():
+    """Курс называет uv до первой команды."""
+    text = (REPO_ROOT / "docs" / "course" / "README.md").read_text(encoding="utf-8")
+
+    assert "uv" in text
+    assert text.index("uv") < text.index("make generated-history-analytics")
+
+
+def test_startup_history_runbook_warns_about_daily_wave_idle_gap():
+    """Runbook предупреждает о дыре модельного времени при простое daily-wave."""
+    text = (REPO_ROOT / "docs" / "runbooks" / "startup-history.md").read_text(
+        encoding="utf-8"
+    )
+
+    assert "daily-wave" in text
+    assert "прост" in text
+    assert "дыр" in text
+    assert "make generator-reset" in text
+    assert "startup-history-import" in text
+
+
+def test_no_docker_or_continue_operation_in_dag():
+    """Пульт не управляет Docker и не содержит операцию continue."""
+    text = DAG_PATH.read_text(encoding="utf-8")
+
+    assert "docker" not in text.lower()
+    assert '"continue"' not in text
+
+
+def test_compose_mounts_generator_code_without_socket_and_gates_live_service():
+    """Airflow видит код генератора, но не получает Docker socket."""
+    text = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
+
+    assert "PYTHONPATH: /opt/airflow/generator_src" in text
+    assert "./generator/src:/opt/airflow/generator_src:ro" in text
+    assert "/var/run/docker.sock" not in text
+    assert "profiles:\n      - live-generator" in text
+
+
+def test_down_and_clean_include_live_generator_profile():
+    """Остановка стенда удаляет профильный live-генератор."""
+    text = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
+
+    assert "down:\n\t$(COMPOSE) --profile live-generator down" in text
+    assert "clean:\n\t$(COMPOSE) --profile live-generator down -v --remove-orphans" in text
+
+
+def test_console_generator_paths_precheck_clean_stand_before_writes():
+    """Backfill/import с хоста проверяют чистый стенд до записи."""
+    run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
+        encoding="utf-8"
+    )
+    import_artifact = (
+        REPO_ROOT / "scripts" / "import_startup_history_artifact.sh"
+    ).read_text(encoding="utf-8")
+
+    assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in run_generator
+    assert run_generator.index("assert_stand_clean.sh") < run_generator.index(
+        "run --rm --no-deps"
+    )
+    assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in import_artifact
+    assert import_artifact.index("assert_stand_clean.sh") < import_artifact.index(
+        "startup_history_artifact import"
+    )
+
+
+def test_reference_artifact_is_import_default_only():
+    """Экспорт не может по умолчанию перезаписать эталонный мир."""
+    makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
+    export_script = (
+        REPO_ROOT / "scripts" / "export_startup_history_artifact.sh"
+    ).read_text(encoding="utf-8")
+    import_script = (
+        REPO_ROOT / "scripts" / "import_startup_history_artifact.sh"
+    ).read_text(encoding="utf-8")
+
+    assert "STARTUP_HISTORY_EXPORT_ARTIFACT" in makefile
+    assert "STARTUP_HISTORY_IMPORT_ARTIFACT" in makefile
+    assert "/tmp/clickstream-startup-history.json" in export_script
+    assert "data/startup_history/reference-world.json.xz" in import_script
+
+
+def test_console_continue_checks_state_or_clean_stand_before_live_start():
+    """Continue не стартует новый live-мир на грязном стенде без state."""
+    run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
+        encoding="utf-8"
+    )
+
+    assert "elif [[ \"${VERB}\" == \"continue\" ]]" in run_generator
+    assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" continue" in run_generator
+    assert run_generator.index("assert_stand_clean.sh\" continue") < run_generator.index(
+        "up -d --build generator"
+    )
+
+
+def test_console_reset_checks_clean_stand_before_live_start():
+    """Reset не пишет новый мир поверх старых данных."""
+    run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
+        encoding="utf-8"
+    )
+    reset_branch = run_generator.split("else\n", maxsplit=1)[1]
+
+    assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in reset_branch
+    assert reset_branch.index("assert_stand_clean.sh\" clean") < reset_branch.index(
+        "up -d --build generator"
+    )
+
+
+def test_clean_start_paths_include_live_generator_profile():
+    """Все чистые сбросы видят профильный live-генератор."""
+    generated_history = (
+        REPO_ROOT / "scripts" / "run_generated_history_analytics.sh"
+    ).read_text(encoding="utf-8")
+
+    assert "--profile live-generator down -v --remove-orphans" in generated_history
+
+
+def test_host_clean_precheck_checks_live_before_kafka_and_stg():
+    """Host backfill/import сначала проверяет, что live-генератор не запущен."""
+    text = (REPO_ROOT / "scripts" / "assert_stand_clean.sh").read_text(
+        encoding="utf-8"
+    )
+
+    assert "--mode live" in text
+    assert text.index("--mode live") < text.index("clickhouse-client")
+    assert text.index("clickhouse-client") < text.index("--mode \"${CHECK_MODE}\"")
+
+
+def test_airflow_and_generator_kafka_dependency_versions_match():
+    """Airflow и генератор используют одну версию kafka-python."""
+    airflow_req = (REPO_ROOT / "airflow" / "requirements.txt").read_text(encoding="utf-8")
+    generator_req = (REPO_ROOT / "generator" / "requirements.txt").read_text(encoding="utf-8")
+
+    assert "kafka-python==2.0.6" in airflow_req
+    assert "kafka-python==2.0.6" in generator_req
+    assert "prometheus-client==0.21.1" in airflow_req