fix(generator): исключено смешение миров при запуске

- Зачем:
  - старый state и ручной live-генератор могли тихо писать события в новый мир данных.
- Что:
  - добавлена общая проверка чистого стенда для Airflow и host-путей.
  - защищены backfill, import, continue и clean-start сценарии live-generator.
  - описан миграционный отказ старого state.
- Проверка:
  - make generator-test; docker compose config --quiet; py_compile; bash -n.
This commit is contained in:
2026-07-05 20:21:13 +03:00
parent 525bd8cabf
commit 080d5ae313
17 changed files with 464 additions and 45 deletions
+42 -2
View File
@@ -68,7 +68,7 @@ def test_assert_stand_clean_rejects_non_empty_stg_before_writes():
def test_assert_stand_clean_rejects_non_empty_kafka_with_make_clean_hint(monkeypatch):
"""Непустые Kafka-топики дают ту же подсказку про make clean."""
from clickstream_generator import airflow_control
from clickstream_generator import airflow_control, stand_clean
class Inspector:
def __init__(self, bootstrap_servers):
@@ -81,12 +81,52 @@ def test_assert_stand_clean_rejects_non_empty_kafka_with_make_clean_hint(monkeyp
def execute(self, sql):
return [(0, 0, 0, 0)]
monkeypatch.setattr(airflow_control, "KafkaTopicInspector", Inspector)
monkeypatch.setattr(stand_clean, "KafkaTopicInspector", Inspector)
with pytest.raises(RuntimeError, match="make clean"):
airflow_control.assert_stand_clean("kafka:29092", Hook())
def test_shared_stand_clean_rejects_host_stg_counts():
"""Хостовая предпроверка использует ту же STG-границу, что пульт."""
from clickstream_generator.stand_clean import assert_stg_counts_empty
with pytest.raises(RuntimeError, match="stg.location_raw=3.*make clean"):
assert_stg_counts_empty((0, 3, 0, 0))
def test_continue_rejects_dirty_stand_without_state(base_config, monkeypatch):
"""Continue без state не стартует новый мир поверх грязного STG."""
from clickstream_generator import stand_clean
class StateManager:
def __init__(self, bootstrap_servers):
self.bootstrap_servers = bootstrap_servers
def load(self):
return None
def close(self):
pass
class Inspector:
def __init__(self, bootstrap_servers):
self.bootstrap_servers = bootstrap_servers
def assert_data_topics_empty(self):
pass
monkeypatch.setattr(stand_clean, "KafkaStateManager", StateManager)
monkeypatch.setattr(stand_clean, "KafkaTopicInspector", Inspector)
with pytest.raises(RuntimeError, match="stg.browser_raw=1.*make clean"):
stand_clean.assert_continue_has_state_or_clean_stand(
"localhost:9092",
(1, 0, 0, 0),
base_config,
)
def test_check_manifest_compares_clickhouse_stats(base_config):
"""Check падает, когда контрольные числа ClickHouse расходятся с manifest."""
from clickstream_generator.airflow_control import assert_clickhouse_matches_manifest