ddmitry/clickstream-ch-kafka-superset-demo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(generator): глагол next-day — следующий модельный день от слепка

Browse Source 
- Зачем:
  - режим кормления стенда порциями: менти триггерит «следующий день»,
    видит полный цикл DWH за один шаг (задача 13, вариант 2 — генерация
    от слепка T_end).
- Что:
  - новый ограниченный режим next-day: восстановление мира из state,
    генерация ровно [T_end, T_end+24h), публикация данные -> state ->
    манифест (манифест — точка фиксации, автоотката нет).
  - операция next-day в DAG generator_control: своя предпроверка границы
    вместо clean-guard, идемпотентность через параметр expected_t_end.
  - цепочка границ — накопительное поле boundaries в манифесте, старый
    формат читается как [T0, T_end]; импорт не изменён.
  - новая проверка цепочки (make generated-history-chain-check): непарные
    счётчики и однородность по каждой границе, явный статус нулевого
    стыка, хвост за границей по всем четырём топикам, литералы в UTC
    с микросекундами.
  - документация OPERATIONS.md: глагол, предпроверка, восстановление
    после сбоя, ограничение retention; в задаче 13 — решения двух слепых
    ревью постановки и кода с аргументами отклонений.
- Проверка:
  - make test: 204 теста генератора + 31 контракт корня, зелёные.
  - make generated-history-chain-check: зелёный, 2 внутренние границы,
    непарные счётчики нулевые; учебный цикл: DM 322 -> 10026 -> 19196
    за два next-day подряд.
  - make generated-history-runtime-check (регрессия задачи 20): зелёный.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
ddadmin
2026-07-12 23:53:48 +03:00
co-authored by Claude Fable 5
parent 540989d358
commit f5dee26eda
17 changed files with 1501 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/tests/test_airflow_control.py
+88
View File
@@ -50,6 +50,94 @@ def test_import_env_requires_artifact_path_and_uses_backfill_contract():
assert env["GEN_HISTORY_DURATION"] == "2d"
def test_next_day_env_uses_world_settings_and_current_manifest_boundary():
"""Next-day восстанавливает настройки мира из manifest, а не из формы DAG."""
from clickstream_generator.airflow_control import build_next_day_env
manifest = {
"gen_seed": 42,
"model_t0": "2026-01-01T00:00:00+00:00",
"model_t_end": "2026-01-03T00:00:00+00:00",
"model_timezone": "UTC",
"launch_profile": "daily-wave",
"generation_settings": {
"tick_seconds": 1,
"lambda_base_per_min": 60,
"jitter_pct": 0,
"min_events_per_tick": 1,
"max_events_per_tick": 1000,
"max_session_events": 30,
"max_active_sessions": 200,
"population_max": 300,
"p_new_user": 0.15,
"min_return_minutes": 30,
"model_time_speed": 60,
},
}
env = build_next_day_env(manifest)
assert env["GEN_RUN_MODE"] == "next-day"
assert env["GEN_STATE_RESET"] == "false"
assert env["GEN_MODEL_T_END"] == "2026-01-03T00:00:00+00:00"
assert env["GEN_MODEL_TIME_SPEED"] == "60"
assert env["GEN_MAX_SESSION_EVENTS"] == "30"
assert env["GEN_DATA_DIR"] == "/opt/airflow/data"
def test_next_day_env_map_covers_every_generation_setting(base_config):
"""Новая настройка генерации не может потеряться между manifest и env."""
from clickstream_generator.airflow_control import NEXT_DAY_SETTING_ENV_KEYS
from clickstream_generator.startup_history_artifact import (
generation_settings_from_config,
)
assert set(NEXT_DAY_SETTING_ENV_KEYS) == set(
generation_settings_from_config(base_config)
)
def test_next_day_precheck_rejects_stale_expected_boundary_with_both_values():
"""Повторный запуск с прежней границей падает до записи данных."""
from clickstream_generator.airflow_control import assert_expected_t_end
with pytest.raises(RuntimeError) as exc_info:
assert_expected_t_end(
"2026-01-03T00:00:00+00:00",
"2026-01-04T00:00:00+00:00",
)
message = str(exc_info.value)
assert "2026-01-03T00:00:00+00:00" in message
assert "2026-01-04T00:00:00+00:00" in message
def test_next_day_precheck_requires_manifest_and_matching_state():
"""Next-day громко отвергает пустой стенд и state не на T_end."""
from clickstream_generator.airflow_control import assert_next_day_snapshot
from test_startup_history_artifact import _state
state = _state()
with pytest.raises(RuntimeError, match="Manifest"):
assert_next_day_snapshot(None, state)
manifest = {
"run_mode": "backfill",
"model_t0": state.model_t0.isoformat(),
"model_t_end": "2026-01-01T02:00:00+00:00",
"gen_seed": state.gen_seed,
"model_timezone": state.model_timezone,
"state_version": state.version,
"state": {
"last_batch_id": state.last_batch_id,
"model_timestamp": "2026-01-01T02:00:00+00:00",
},
"generation_settings": {"model_time_speed": state.model_time_speed},
}
with pytest.raises(RuntimeError, match="state.*T_end"):
assert_next_day_snapshot(manifest, state)
def test_target_dag_trigger_error_covers_missing_paused_and_ready_states():
"""Проверка зависимого DAG различает три реальные ветки."""
from clickstream_generator.airflow_control import target_dag_trigger_error
generator/tests/test_generator_control_dag_contract.py
+22 -2
View File
@@ -30,14 +30,15 @@ def test_trigger_form_has_expected_param_enums():
"""Форма запуска ограничивает операции и профили."""
text = DAG_PATH.read_text(encoding="utf-8")
assert 'enum=["backfill", "import", "check"]' in text
assert 'enum=["backfill", "import", "next-day", "check"]' in text
assert "enum=sorted(PROFILES)" in text
assert '"duration": Param(' in text
assert '"artifact_path": Param(' in text
assert '"expected_t_end": Param(' in text
def test_dag_branches_and_waits_for_etl_completion():
"""Backfill/import запускают ETL и ждут его завершения перед check."""
"""Backfill/import/next-day запускают ETL и ждут завершения перед check."""
text = DAG_PATH.read_text(encoding="utf-8")
assert "BranchPythonOperator" in text
@@ -48,6 +49,23 @@ def test_dag_branches_and_waits_for_etl_completion():
assert 'failed_states=["failed"]' in text
def test_next_day_has_own_boundary_precheck_and_serial_execution():
"""Next-day не использует clean-guard и не допускает параллельных запусков."""
text = DAG_PATH.read_text(encoding="utf-8")
assert "schedule=None" in text
assert "max_active_runs=1" in text
assert 'return "check_etl_not_paused_before_next_day"' in text
assert "assert_next_day_snapshot" in text
assert "assert_expected_t_end" in text
next_day_precheck = text.split("def precheck_next_day", maxsplit=1)[1].split(
"\ndef ", maxsplit=1
)[0]
assert "assert_stand_clean" not in next_day_precheck
assert "assert_live_generator_not_running" in next_day_precheck
assert "run_next_day" in text
def test_generator_control_prechecks_etl_dag_not_paused_before_waiting():
"""Пульт проверяет паузу etl_pipeline до долгого ожидания."""
text = DAG_PATH.read_text(encoding="utf-8")
@@ -62,8 +80,10 @@ def test_generator_control_prechecks_etl_dag_not_paused_before_waiting():
assert "fail_when_dag_is_paused" in text
assert 'return "check_etl_not_paused_before_backfill"' in text
assert 'return "check_etl_not_paused_before_import"' in text
assert 'return "check_etl_not_paused_before_next_day"' in text
assert "check_etl_not_paused_before_backfill >> precheck_backfill_task" in text
assert "check_etl_not_paused_before_import >> precheck_import_task" in text
assert "check_etl_not_paused_before_next_day >> precheck_next_day_task" in text
assert text.index("check_etl_not_paused_before_backfill >> precheck_backfill_task") < text.index(
"precheck_backfill_task >> backfill_task"
)
generator/tests/test_launch.py
+23 -1
View File
@@ -43,7 +43,11 @@ def test_continue_and_reset_verbs_map_to_existing_low_level_flags():
from clickstream_generator.launch import build_launch_env
continue_env = build_launch_env("continue", profile_name="ci")
reset_env = build_launch_env("reset", profile_name="ci")
reset_env = build_launch_env(
"reset",
profile_name="ci",
overrides={"GEN_MODEL_T_END": "2026-01-09T00:00:00+00:00"},
)
assert continue_env["GEN_RUN_MODE"] == "live"
assert continue_env["GEN_STATE_RESET"] == "false"
@@ -53,6 +57,24 @@ def test_continue_and_reset_verbs_map_to_existing_low_level_flags():
assert "GEN_MODEL_T_END" not in reset_env
def test_next_day_verb_restores_snapshot_without_reset():
"""Глагол next-day включает отдельный ограниченный режим от текущей границы."""
from clickstream_generator.launch import build_launch_env
env = build_launch_env(
"next-day",
profile_name="ci",
overrides={
"GEN_MODEL_T0": "2026-01-01T00:00:00+00:00",
"GEN_MODEL_T_END": "2026-01-03T00:00:00+00:00",
},
)
assert env["GEN_RUN_MODE"] == "next-day"
assert env["GEN_STATE_RESET"] == "false"
assert env["GEN_MODEL_T_END"] == "2026-01-03T00:00:00+00:00"
def test_ci_profile_keeps_previous_default_backfill_window():
"""CI-профиль сохраняет прежний 6-часовой проверочный запуск."""
from clickstream_generator.launch import build_launch_env
generator/tests/test_service.py
+179
View File
@@ -1005,6 +1005,185 @@ class TestGeneratorServiceBackfill:
}
class TestGeneratorServiceNextDay:
"""Проверки ограниченной доливки следующего модельного дня."""
def test_next_day_publishes_24h_then_state_then_cumulative_manifest(
self, base_config
):
"""Next-day пишет [T_end, T_end+24h), затем state и manifest."""
from clickstream_generator.startup_history_artifact import (
StartupHistoryArtifactBuilder,
build_manifest,
)
model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
current_t_end = model_t0 + timedelta(hours=1)
config = replace(
base_config,
run_mode="next-day",
model_t0=model_t0,
model_t_end=current_t_end,
tick_seconds=3600,
model_time_speed=1,
lambda_base_per_min=60,
jitter_pct=0,
min_events_per_tick=1,
max_events_per_tick=1000,
max_session_events=5,
max_active_sessions=250,
population_max=251,
)
service = GeneratorService(config)
state = service.stream.to_state(
tick=1,
rng_state=service.generator.rng.getstate(),
last_batch_id="startup-history-initial",
last_timestamp=current_t_end,
model_timestamp=current_t_end,
wall_timestamp=current_t_end,
model_time_speed=config.model_time_speed,
model_timezone=config.model_timezone,
model_t0=config.model_t0,
gen_seed=config.seed,
)
old_batch = {
"browser_events": [{
"event_id": "old-event",
"click_id": "old-click",
"event_timestamp": "2026-01-01 00:30:00.000000",
}],
"location_events": [{"event_id": "old-event"}],
"device_events": [{
"click_id": "old-click",
"user_domain_id": "old-user",
}],
"geo_events": [{"click_id": "old-click"}],
}
initial_builder = StartupHistoryArtifactBuilder()
initial_builder.add_batch(old_batch)
manifest = build_manifest(config, initial_builder.counters, state)
published = {topic: [] for topic in old_batch}
service.publisher = MagicMock()
def publish(topic, events):
published[topic].extend(events)
return len(events), 0
service.publisher.publish.side_effect = publish
service.history = MagicMock()
order = []
service.publisher.flush.side_effect = lambda: order.append("data")
service.state_manager = MagicMock()
service.state_manager.save.side_effect = lambda _state: order.append("state")
service.manifest_manager = MagicMock()
service.manifest_manager.save.side_effect = (
lambda _manifest: order.append("manifest")
)
class DataReader:
def load(self):
return {
topic: old_batch[topic] + published[topic]
for topic in old_batch
}
service.data_reader = DataReader()
service.restore_from_startup_history(state, model_t_end=current_t_end)
service._run_next_day(manifest)
browser_events = published["browser_events"]
timestamps = [
datetime.fromisoformat(event["event_timestamp"].replace(" ", "T"))
for event in browser_events
]
target_t_end = current_t_end + timedelta(hours=24)
saved_state = service.state_manager.save.call_args.args[0]
saved_manifest = service.manifest_manager.save.call_args.args[0]
assert browser_events
assert min(timestamps) >= current_t_end.replace(tzinfo=None)
assert max(timestamps) < target_t_end.replace(tzinfo=None)
assert saved_state.model_timestamp == target_t_end
assert saved_manifest["model_t_end"] == target_t_end.isoformat()
assert saved_manifest["boundaries"] == [
model_t0.isoformat(),
current_t_end.isoformat(),
target_t_end.isoformat(),
]
assert saved_manifest["totals"]["events"] == len(browser_events) + 1
assert order == ["data", "state", "manifest"]
first_day_events = len(browser_events)
first_day_checksum = saved_manifest["topics"]["browser_events"][
"checksum_sha256"
]
service._run_next_day(saved_manifest)
second_state = service.state_manager.save.call_args.args[0]
second_manifest = service.manifest_manager.save.call_args.args[0]
expected_builder = StartupHistoryArtifactBuilder()
expected_builder.add_batch(DataReader().load())
assert second_state.model_timestamp == target_t_end + timedelta(hours=24)
assert second_manifest["boundaries"] == [
model_t0.isoformat(),
current_t_end.isoformat(),
target_t_end.isoformat(),
(target_t_end + timedelta(hours=24)).isoformat(),
]
assert second_manifest["totals"]["events"] == len(browser_events) + 1
assert len(browser_events) > first_day_events
assert (
second_manifest["topics"]["browser_events"]["checksum_sha256"]
!= first_day_checksum
)
assert second_manifest["topics"] == expected_builder.counters.to_manifest_topics()
assert order == [
"data",
"state",
"manifest",
"data",
"state",
"manifest",
]
def test_next_day_publish_error_does_not_move_state_or_manifest(self, base_config):
"""Ошибка data-топика оставляет обе точки фиксации без изменений."""
model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
config = replace(
base_config,
run_mode="next-day",
model_t0=model_t0,
model_t_end=model_t0 + timedelta(hours=1),
tick_seconds=3600,
model_time_speed=1,
max_active_sessions=250,
population_max=251,
)
service = GeneratorService(config)
service.publisher = MagicMock()
service.publisher.publish.side_effect = lambda topic, events: (
(len(events), 1) if topic == "location_events" else (len(events), 0)
)
service.history = MagicMock()
service.state_manager = MagicMock()
service.manifest_manager = MagicMock()
service._model_time = config.model_t_end
manifest = {
"model_t0": model_t0.isoformat(),
"model_t_end": config.model_t_end.isoformat(),
"boundaries": [model_t0.isoformat(), config.model_t_end.isoformat()],
}
with pytest.raises(RuntimeError, match="Публикация next-day не удалась"):
service._run_next_day(manifest)
service.state_manager.save.assert_not_called()
service.manifest_manager.save.assert_not_called()
class TestGeneratorServiceState:
"""Тесты подключения state к сервисному запуску."""
generator/tests/test_startup_history_artifact.py
+86
View File
@@ -167,6 +167,92 @@ def test_manifest_and_artifact_show_launch_profile(base_config):
path.unlink(missing_ok=True)
def test_manifest_records_initial_boundary_chain(base_config):
"""Новый manifest явно хранит начало и правую границу истории."""
from clickstream_generator.startup_history_artifact import (
StartupHistoryArtifactBuilder,
build_manifest,
manifest_boundaries,
)
state = _state()
builder = StartupHistoryArtifactBuilder()
builder.add_batch(_batch())
manifest = build_manifest(
config=replace(
base_config,
model_t0=state.model_t0,
model_t_end=state.model_timestamp,
model_time_speed=1,
model_timezone="UTC",
seed=42,
),
counters=builder.counters,
state=state,
)
assert manifest["boundaries"] == [
"2026-01-01T00:00:00+00:00",
"2026-01-01T01:00:00+00:00",
]
assert manifest_boundaries(manifest) == manifest["boundaries"]
def test_legacy_manifest_without_boundaries_gets_endpoint_chain():
"""Старый manifest без boundaries читается как пара [T0, T_end]."""
from clickstream_generator.startup_history_artifact import manifest_boundaries
manifest = {
"model_t0": "2026-01-01T00:00:00+00:00",
"model_t_end": "2026-01-03T00:00:00+00:00",
}
assert manifest_boundaries(manifest) == [
"2026-01-01T00:00:00+00:00",
"2026-01-03T00:00:00+00:00",
]
def test_manifest_rejects_boundary_chain_with_wrong_endpoint():
"""Цепочка не может расходиться с текущим model_t_end."""
from clickstream_generator.startup_history_artifact import manifest_boundaries
manifest = {
"model_t0": "2026-01-01T00:00:00+00:00",
"model_t_end": "2026-01-03T00:00:00+00:00",
"boundaries": [
"2026-01-01T00:00:00+00:00",
"2026-01-02T00:00:00+00:00",
],
}
with pytest.raises(ValueError, match="model_t_end"):
manifest_boundaries(manifest)
def test_build_manifest_rejects_explicit_empty_boundaries(base_config):
"""Явно пустая цепочка не подменяется границами по умолчанию."""
from clickstream_generator.startup_history_artifact import (
StartupHistoryArtifactBuilder,
build_manifest,
)
state = _state()
builder = StartupHistoryArtifactBuilder()
builder.add_batch(_batch())
config = replace(
base_config,
model_t0=state.model_t0,
model_t_end=state.model_timestamp,
model_time_speed=1,
model_timezone="UTC",
seed=42,
)
with pytest.raises(ValueError, match="T0 и T_end"):
build_manifest(config, builder.counters, state, boundaries=[])
def test_artifact_validation_rejects_mismatched_manifest(base_config):
"""Валидация отвергает артефакт, где manifest не совпадает с событиями."""
from clickstream_generator.startup_history_artifact import (