ddmitry/clickstream-ch-kafka-superset-demo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(airflow): пульт стал world_init, добавлен DAG world_next_day (#4)

Browse Source 
- Зачем:
  - список DAG'ов должен читаться лесенкой ddl_init → world_init →
    world_next_day, а путь менти — проходиться пустыми формами
    (issue #4, спека редизайна пути менти, решения 2–3).
- Что:
  - generator_control переименован в world_init, дефолт операции —
    import; next-day ушёл из выпадашки в отдельный DAG;
  - новый беспараметрный world_next_day: расписание */30 * * * *,
    создаётся на паузе, catchup=False, max_active_runs=1; общие
    задачи вынесены в airflow/dags/utils/startup_history_tasks.py;
  - доки и контрактные тесты обновлены синхронно; быстрый старт
    README — без make ddl, схему создаёт DAG ddl_init.
- Проверка:
  - make test (210 + 31) и make lint зелёные;
  - живая приёмка на чистом стенде: world_init пустой формой
    импортировал эталонный мир за 217 с (3 дня, 280 437 событий),
    world_next_day после снятия с паузы добавляет ровно один день
    за прогон, дашборд Superset собирается.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
ddadmin
2026-07-22 21:45:02 +03:00
co-authored by Claude Fable 5
parent 76ff669cf7
commit ac7a973504
12 changed files with 268 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/src/clickstream_generator/airflow_control.py
+1 -1
View File
@@ -328,7 +328,7 @@ def target_dag_trigger_error(dag_id: str, dag_model) -> str | None:
if dag_model.is_paused:
return (
f"DAG {dag_id} стоит на паузе: снимите паузу в Airflow UI, "
"затем повторите generator_control."
"затем повторите запуск DAG."
)
return None
generator/tests/test_generator_control_dag_contract.py → generator/tests/test_world_dags_contract.py
+36 -34
View File
@@ -1,6 +1,4 @@
"""
Контракт DAG generator_control без запуска Airflow.
"""
"""Контракты DAG world_init и world_next_day без запуска Airflow."""
import ast
from pathlib import Path
@@ -8,7 +6,8 @@ from pathlib import Path
from clickstream_generator.launch import PROFILES
DAG_PATH = Path(__file__).parents[2] / "airflow" / "dags" / "generator_control_dag.py"
DAG_PATH = Path(__file__).parents[2] / "airflow" / "dags" / "world_init_dag.py"
NEXT_DAY_DAG_PATH = Path(__file__).parents[2] / "airflow" / "dags" / "world_next_day_dag.py"
REPO_ROOT = Path(__file__).parents[2]
@@ -65,7 +64,7 @@ def test_dag_file_exists_and_uses_dynamic_profiles():
"""DAG берёт варианты профилей из PROFILES, а не из ручного списка."""
text = DAG_PATH.read_text(encoding="utf-8")
assert "dag_id=\"generator_control\"" in text
assert "dag_id=\"world_init\"" in text
assert "sorted(PROFILES)" in text
for profile in PROFILES:
assert profile not in {"hardcoded-profile"}
@@ -76,12 +75,13 @@ def test_trigger_form_has_expected_param_enums():
text = DAG_PATH.read_text(encoding="utf-8")
param_defaults = _declared_param_defaults()
assert 'enum=["backfill", "import", "next-day", "check"]' in text
assert 'enum=["backfill", "import", "check"]' in text
assert param_defaults["operation"] == "import"
assert "enum=sorted(PROFILES)" in text
assert param_defaults["profile"] == "daily-wave"
assert '"duration": Param(' in text
assert '"artifact_path": Param(' in text
assert '"expected_t_end": Param(' in text
assert '"expected_t_end": Param(' not in text
def test_trigger_form_marks_only_optional_params_as_nullable():
@@ -93,7 +93,6 @@ def test_trigger_form_marks_only_optional_params_as_nullable():
"seed",
"model_time_speed",
"artifact_path",
"expected_t_end",
}:
assert "null" in param_types[name]
for name in {"operation", "profile"}:
@@ -101,7 +100,7 @@ def test_trigger_form_marks_only_optional_params_as_nullable():
def test_dag_branches_and_waits_for_etl_completion():
"""Backfill/import/next-day запускают ETL и ждут завершения перед check."""
"""Backfill/import запускают ETL и ждут завершения перед check."""
text = DAG_PATH.read_text(encoding="utf-8")
assert "BranchPythonOperator" in text
@@ -112,41 +111,43 @@ def test_dag_branches_and_waits_for_etl_completion():
assert 'failed_states=["failed"]' in text
def test_next_day_has_own_boundary_precheck_and_serial_execution():
"""Next-day не использует clean-guard и не допускает параллельных запусков."""
text = DAG_PATH.read_text(encoding="utf-8")
def test_world_next_day_is_parameterless_paused_half_hour_dag():
"""DAG следующего дня запускается пустой формой каждые полчаса."""
text = NEXT_DAY_DAG_PATH.read_text(encoding="utf-8")
assert "schedule=None" in text
assert 'dag_id="world_next_day"' in text
assert 'schedule="*/30 * * * *"' in text
assert "catchup=False" in text
assert "is_paused_upon_creation=True" in text
assert "max_active_runs=1" in text
assert 'return "check_etl_not_paused_before_next_day"' in text
assert "assert_next_day_snapshot" in text
assert "assert_expected_t_end" in text
next_day_precheck = text.split("def precheck_next_day", maxsplit=1)[1].split(
"\ndef ", maxsplit=1
)[0]
assert "assert_stand_clean" not in next_day_precheck
assert "assert_live_generator_not_running" in next_day_precheck
assert "run_next_day" in text
assert "params=" not in text
assert "precheck_next_day" in text
assert "run_next_day_task" in text
assert "TriggerDagRunOperator" in text
assert "wait_for_completion=True" in text
assert "check_manifest_task" in text
assert "check_etl_not_paused >> precheck" in text
def test_generator_control_prechecks_etl_dag_not_paused_before_waiting():
def test_world_dags_precheck_etl_dag_not_paused_before_waiting():
"""Пульт проверяет паузу etl_pipeline до долгого ожидания."""
text = DAG_PATH.read_text(encoding="utf-8")
shared_tasks = (
REPO_ROOT / "airflow" / "dags" / "utils" / "startup_history_tasks.py"
).read_text(encoding="utf-8")
assert "assert_target_dag_not_paused" in text
assert 'ETL_DAG_ID = "etl_pipeline"' in text
assert 'op_kwargs={"dag_id": ETL_DAG_ID}' in text
assert "session.query(DagModel)" in text
assert "DagModel.dag_id == dag_id" in text
assert "target_dag_trigger_error" in text
assert "Airflow 2.10.5" in text
assert "fail_when_dag_is_paused" in text
assert "session.query(DagModel)" in shared_tasks
assert "DagModel.dag_id == dag_id" in shared_tasks
assert "target_dag_trigger_error" in shared_tasks
assert "Airflow 2.10.5" in shared_tasks
assert "fail_when_dag_is_paused" in shared_tasks
assert 'return "check_etl_not_paused_before_backfill"' in text
assert 'return "check_etl_not_paused_before_import"' in text
assert 'return "check_etl_not_paused_before_next_day"' in text
assert "check_etl_not_paused_before_backfill >> precheck_backfill_task" in text
assert "check_etl_not_paused_before_import >> precheck_import_task" in text
assert "check_etl_not_paused_before_next_day >> precheck_next_day_task" in text
assert text.index("check_etl_not_paused_before_backfill >> precheck_backfill_task") < text.index(
"precheck_backfill_task >> backfill_task"
)
@@ -170,15 +171,16 @@ def test_make_up_rebuilds_airflow_images_after_repo_update():
assert "superset-init" not in up_block
def test_readme_quick_start_lists_prerequisites_and_runs_ddl_before_generator_control():
"""Быстрый старт называет инструменты и DDL до generator_control."""
def test_readme_quick_start_creates_schema_via_ddl_init_before_world_init():
"""Быстрый старт: терминал — только make up, схему создаёт ddl_init до world_init."""
text = (REPO_ROOT / "README.md").read_text(encoding="utf-8")
assert "uv" in text
assert "Docker" in text
assert "docker compose" in text
assert "make up\nmake ddl" in text
assert text.index("make ddl") < text.index("generator_control")
assert "make ddl" not in text
quick_start = text.split("## Быстрый старт", maxsplit=1)[1]
assert quick_start.index("ddl_init") < quick_start.index("world_init")
def test_course_readme_lists_uv_before_first_command():