fix(generator): исключено смешение миров при запуске

- Зачем:
  - старый state и ручной live-генератор могли тихо писать события в новый мир данных.
- Что:
  - добавлена общая проверка чистого стенда для Airflow и host-путей.
  - защищены backfill, import, continue и clean-start сценарии live-generator.
  - описан миграционный отказ старого state.
- Проверка:
  - make generator-test; docker compose config --quiet; py_compile; bash -n.
This commit is contained in:
2026-07-05 20:21:13 +03:00
parent 525bd8cabf
commit 080d5ae313
17 changed files with 464 additions and 45 deletions
+42 -2
View File
@@ -68,7 +68,7 @@ def test_assert_stand_clean_rejects_non_empty_stg_before_writes():
def test_assert_stand_clean_rejects_non_empty_kafka_with_make_clean_hint(monkeypatch):
"""Непустые Kafka-топики дают ту же подсказку про make clean."""
from clickstream_generator import airflow_control
from clickstream_generator import airflow_control, stand_clean
class Inspector:
def __init__(self, bootstrap_servers):
@@ -81,12 +81,52 @@ def test_assert_stand_clean_rejects_non_empty_kafka_with_make_clean_hint(monkeyp
def execute(self, sql):
return [(0, 0, 0, 0)]
monkeypatch.setattr(airflow_control, "KafkaTopicInspector", Inspector)
monkeypatch.setattr(stand_clean, "KafkaTopicInspector", Inspector)
with pytest.raises(RuntimeError, match="make clean"):
airflow_control.assert_stand_clean("kafka:29092", Hook())
def test_shared_stand_clean_rejects_host_stg_counts():
"""Хостовая предпроверка использует ту же STG-границу, что пульт."""
from clickstream_generator.stand_clean import assert_stg_counts_empty
with pytest.raises(RuntimeError, match="stg.location_raw=3.*make clean"):
assert_stg_counts_empty((0, 3, 0, 0))
def test_continue_rejects_dirty_stand_without_state(base_config, monkeypatch):
"""Continue без state не стартует новый мир поверх грязного STG."""
from clickstream_generator import stand_clean
class StateManager:
def __init__(self, bootstrap_servers):
self.bootstrap_servers = bootstrap_servers
def load(self):
return None
def close(self):
pass
class Inspector:
def __init__(self, bootstrap_servers):
self.bootstrap_servers = bootstrap_servers
def assert_data_topics_empty(self):
pass
monkeypatch.setattr(stand_clean, "KafkaStateManager", StateManager)
monkeypatch.setattr(stand_clean, "KafkaTopicInspector", Inspector)
with pytest.raises(RuntimeError, match="stg.browser_raw=1.*make clean"):
stand_clean.assert_continue_has_state_or_clean_stand(
"localhost:9092",
(1, 0, 0, 0),
base_config,
)
def test_check_manifest_compares_clickhouse_stats(base_config):
"""Check падает, когда контрольные числа ClickHouse расходятся с manifest."""
from clickstream_generator.airflow_control import assert_clickhouse_matches_manifest
@@ -66,6 +66,66 @@ def test_compose_mounts_generator_code_without_socket_and_gates_live_service():
assert "profiles:\n - live-generator" in text
def test_down_and_clean_include_live_generator_profile():
"""Остановка стенда удаляет профильный live-генератор."""
text = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
assert "down:\n\t$(COMPOSE) --profile live-generator down" in text
assert "clean:\n\t$(COMPOSE) --profile live-generator down -v --remove-orphans" in text
def test_console_generator_paths_precheck_clean_stand_before_writes():
"""Backfill/import с хоста проверяют чистый стенд до записи."""
run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
encoding="utf-8"
)
import_artifact = (
REPO_ROOT / "scripts" / "import_startup_history_artifact.sh"
).read_text(encoding="utf-8")
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in run_generator
assert run_generator.index("assert_stand_clean.sh") < run_generator.index(
"run --rm --no-deps"
)
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in import_artifact
assert import_artifact.index("assert_stand_clean.sh") < import_artifact.index(
"startup_history_artifact import"
)
def test_console_continue_checks_state_or_clean_stand_before_live_start():
"""Continue не стартует новый live-мир на грязном стенде без state."""
run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
encoding="utf-8"
)
assert "elif [[ \"${VERB}\" == \"continue\" ]]" in run_generator
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" continue" in run_generator
assert run_generator.index("assert_stand_clean.sh\" continue") < run_generator.index(
"up -d --build generator"
)
def test_clean_start_paths_include_live_generator_profile():
"""Все чистые сбросы видят профильный live-генератор."""
generated_history = (
REPO_ROOT / "scripts" / "run_generated_history_analytics.sh"
).read_text(encoding="utf-8")
assert "--profile live-generator down -v --remove-orphans" in generated_history
def test_host_clean_precheck_checks_live_before_kafka_and_stg():
"""Host backfill/import сначала проверяет, что live-генератор не запущен."""
text = (REPO_ROOT / "scripts" / "assert_stand_clean.sh").read_text(
encoding="utf-8"
)
assert "--mode live" in text
assert text.index("--mode live") < text.index("clickhouse-client")
assert text.index("clickhouse-client") < text.index("--mode \"${CHECK_MODE}\"")
def test_airflow_and_generator_kafka_dependency_versions_match():
"""Airflow и генератор используют одну версию kafka-python."""
airflow_req = (REPO_ROOT / "airflow" / "requirements.txt").read_text(encoding="utf-8")
+26
View File
@@ -22,6 +22,7 @@ from generator import (
KafkaBatchHistory,
TickStreamGenerator,
)
from clickstream_generator.state import UnsupportedStateVersionError
class TestBatchRecordWithDictConversion:
@@ -1162,6 +1163,31 @@ class TestGeneratorServiceState:
state_manager.load.assert_not_called()
assert service._tick == 0
def test_known_old_state_version_fails_instead_of_starting_new_world(
self, base_config, caplog
):
"""GEN_STATE_RESET=false не скрывает старый state за чистым стартом."""
state_manager = MagicMock()
state_manager.load.side_effect = UnsupportedStateVersionError("2.0", "3.0")
with patch("clickstream_generator.service.start_http_server"), \
patch("clickstream_generator.service.ensure_topics"), \
patch("clickstream_generator.service.KafkaPublisher"), \
patch("clickstream_generator.service.KafkaBatchHistory"), \
patch(
"clickstream_generator.service.KafkaStateManager",
return_value=state_manager,
), \
patch.object(GeneratorService, "_main_loop", return_value=None), \
caplog.at_level(logging.ERROR, logger="generator"):
service = GeneratorService(base_config)
with pytest.raises(ValueError, match="state version 2.0.*GEN_STATE_RESET=true"):
service.start()
assert service._tick == 0
assert "Readable generator state uses an old format" in caplog.text
def test_invalid_restored_state_starts_fresh(self, base_config, caplog):
"""Сервис не падает, если state ссылается на неизвестный профиль."""
state_manager = MagicMock()
+9
View File
@@ -10,6 +10,7 @@ from unittest.mock import MagicMock, patch
import pytest
from generator import GeneratorState, KafkaStateManager
from clickstream_generator.state import UnsupportedStateVersionError
def _make_valid_rng_state(seed: int = 42):
@@ -423,6 +424,14 @@ class TestGeneratorStateValidation:
with pytest.raises(ValueError, match="version"):
GeneratorState.from_dict(data)
def test_from_dict_rejects_known_old_state_version_loudly(self):
"""Известная старая версия state даёт отдельную ошибку миграции."""
data = _make_valid_state_data()
data["version"] = "2.0"
with pytest.raises(UnsupportedStateVersionError, match="2.0.*3.0"):
GeneratorState.from_dict(data)
class TestKafkaStateManager:
"""Тесты менеджера состояния."""