fix(generator): исключено смешение миров при запуске
- Зачем: - старый state и ручной live-генератор могли тихо писать события в новый мир данных. - Что: - добавлена общая проверка чистого стенда для Airflow и host-путей. - защищены backfill, import, continue и clean-start сценарии live-generator. - описан миграционный отказ старого state. - Проверка: - make generator-test; docker compose config --quiet; py_compile; bash -n.
This commit is contained in:
@@ -68,7 +68,7 @@ def test_assert_stand_clean_rejects_non_empty_stg_before_writes():
|
||||
|
||||
def test_assert_stand_clean_rejects_non_empty_kafka_with_make_clean_hint(monkeypatch):
|
||||
"""Непустые Kafka-топики дают ту же подсказку про make clean."""
|
||||
from clickstream_generator import airflow_control
|
||||
from clickstream_generator import airflow_control, stand_clean
|
||||
|
||||
class Inspector:
|
||||
def __init__(self, bootstrap_servers):
|
||||
@@ -81,12 +81,52 @@ def test_assert_stand_clean_rejects_non_empty_kafka_with_make_clean_hint(monkeyp
|
||||
def execute(self, sql):
|
||||
return [(0, 0, 0, 0)]
|
||||
|
||||
monkeypatch.setattr(airflow_control, "KafkaTopicInspector", Inspector)
|
||||
monkeypatch.setattr(stand_clean, "KafkaTopicInspector", Inspector)
|
||||
|
||||
with pytest.raises(RuntimeError, match="make clean"):
|
||||
airflow_control.assert_stand_clean("kafka:29092", Hook())
|
||||
|
||||
|
||||
def test_shared_stand_clean_rejects_host_stg_counts():
|
||||
"""Хостовая предпроверка использует ту же STG-границу, что пульт."""
|
||||
from clickstream_generator.stand_clean import assert_stg_counts_empty
|
||||
|
||||
with pytest.raises(RuntimeError, match="stg.location_raw=3.*make clean"):
|
||||
assert_stg_counts_empty((0, 3, 0, 0))
|
||||
|
||||
|
||||
def test_continue_rejects_dirty_stand_without_state(base_config, monkeypatch):
|
||||
"""Continue без state не стартует новый мир поверх грязного STG."""
|
||||
from clickstream_generator import stand_clean
|
||||
|
||||
class StateManager:
|
||||
def __init__(self, bootstrap_servers):
|
||||
self.bootstrap_servers = bootstrap_servers
|
||||
|
||||
def load(self):
|
||||
return None
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
class Inspector:
|
||||
def __init__(self, bootstrap_servers):
|
||||
self.bootstrap_servers = bootstrap_servers
|
||||
|
||||
def assert_data_topics_empty(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(stand_clean, "KafkaStateManager", StateManager)
|
||||
monkeypatch.setattr(stand_clean, "KafkaTopicInspector", Inspector)
|
||||
|
||||
with pytest.raises(RuntimeError, match="stg.browser_raw=1.*make clean"):
|
||||
stand_clean.assert_continue_has_state_or_clean_stand(
|
||||
"localhost:9092",
|
||||
(1, 0, 0, 0),
|
||||
base_config,
|
||||
)
|
||||
|
||||
|
||||
def test_check_manifest_compares_clickhouse_stats(base_config):
|
||||
"""Check падает, когда контрольные числа ClickHouse расходятся с manifest."""
|
||||
from clickstream_generator.airflow_control import assert_clickhouse_matches_manifest
|
||||
|
||||
@@ -66,6 +66,66 @@ def test_compose_mounts_generator_code_without_socket_and_gates_live_service():
|
||||
assert "profiles:\n - live-generator" in text
|
||||
|
||||
|
||||
def test_down_and_clean_include_live_generator_profile():
|
||||
"""Остановка стенда удаляет профильный live-генератор."""
|
||||
text = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
|
||||
|
||||
assert "down:\n\t$(COMPOSE) --profile live-generator down" in text
|
||||
assert "clean:\n\t$(COMPOSE) --profile live-generator down -v --remove-orphans" in text
|
||||
|
||||
|
||||
def test_console_generator_paths_precheck_clean_stand_before_writes():
|
||||
"""Backfill/import с хоста проверяют чистый стенд до записи."""
|
||||
run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
import_artifact = (
|
||||
REPO_ROOT / "scripts" / "import_startup_history_artifact.sh"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in run_generator
|
||||
assert run_generator.index("assert_stand_clean.sh") < run_generator.index(
|
||||
"run --rm --no-deps"
|
||||
)
|
||||
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" clean" in import_artifact
|
||||
assert import_artifact.index("assert_stand_clean.sh") < import_artifact.index(
|
||||
"startup_history_artifact import"
|
||||
)
|
||||
|
||||
|
||||
def test_console_continue_checks_state_or_clean_stand_before_live_start():
|
||||
"""Continue не стартует новый live-мир на грязном стенде без state."""
|
||||
run_generator = (REPO_ROOT / "scripts" / "run_generator.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "elif [[ \"${VERB}\" == \"continue\" ]]" in run_generator
|
||||
assert "bash \"${SCRIPT_DIR}/assert_stand_clean.sh\" continue" in run_generator
|
||||
assert run_generator.index("assert_stand_clean.sh\" continue") < run_generator.index(
|
||||
"up -d --build generator"
|
||||
)
|
||||
|
||||
|
||||
def test_clean_start_paths_include_live_generator_profile():
|
||||
"""Все чистые сбросы видят профильный live-генератор."""
|
||||
generated_history = (
|
||||
REPO_ROOT / "scripts" / "run_generated_history_analytics.sh"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "--profile live-generator down -v --remove-orphans" in generated_history
|
||||
|
||||
|
||||
def test_host_clean_precheck_checks_live_before_kafka_and_stg():
|
||||
"""Host backfill/import сначала проверяет, что live-генератор не запущен."""
|
||||
text = (REPO_ROOT / "scripts" / "assert_stand_clean.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "--mode live" in text
|
||||
assert text.index("--mode live") < text.index("clickhouse-client")
|
||||
assert text.index("clickhouse-client") < text.index("--mode \"${CHECK_MODE}\"")
|
||||
|
||||
|
||||
def test_airflow_and_generator_kafka_dependency_versions_match():
|
||||
"""Airflow и генератор используют одну версию kafka-python."""
|
||||
airflow_req = (REPO_ROOT / "airflow" / "requirements.txt").read_text(encoding="utf-8")
|
||||
|
||||
@@ -22,6 +22,7 @@ from generator import (
|
||||
KafkaBatchHistory,
|
||||
TickStreamGenerator,
|
||||
)
|
||||
from clickstream_generator.state import UnsupportedStateVersionError
|
||||
|
||||
|
||||
class TestBatchRecordWithDictConversion:
|
||||
@@ -1162,6 +1163,31 @@ class TestGeneratorServiceState:
|
||||
state_manager.load.assert_not_called()
|
||||
assert service._tick == 0
|
||||
|
||||
def test_known_old_state_version_fails_instead_of_starting_new_world(
|
||||
self, base_config, caplog
|
||||
):
|
||||
"""GEN_STATE_RESET=false не скрывает старый state за чистым стартом."""
|
||||
state_manager = MagicMock()
|
||||
state_manager.load.side_effect = UnsupportedStateVersionError("2.0", "3.0")
|
||||
|
||||
with patch("clickstream_generator.service.start_http_server"), \
|
||||
patch("clickstream_generator.service.ensure_topics"), \
|
||||
patch("clickstream_generator.service.KafkaPublisher"), \
|
||||
patch("clickstream_generator.service.KafkaBatchHistory"), \
|
||||
patch(
|
||||
"clickstream_generator.service.KafkaStateManager",
|
||||
return_value=state_manager,
|
||||
), \
|
||||
patch.object(GeneratorService, "_main_loop", return_value=None), \
|
||||
caplog.at_level(logging.ERROR, logger="generator"):
|
||||
|
||||
service = GeneratorService(base_config)
|
||||
with pytest.raises(ValueError, match="state version 2.0.*GEN_STATE_RESET=true"):
|
||||
service.start()
|
||||
|
||||
assert service._tick == 0
|
||||
assert "Readable generator state uses an old format" in caplog.text
|
||||
|
||||
def test_invalid_restored_state_starts_fresh(self, base_config, caplog):
|
||||
"""Сервис не падает, если state ссылается на неизвестный профиль."""
|
||||
state_manager = MagicMock()
|
||||
|
||||
@@ -10,6 +10,7 @@ from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
|
||||
from generator import GeneratorState, KafkaStateManager
|
||||
from clickstream_generator.state import UnsupportedStateVersionError
|
||||
|
||||
|
||||
def _make_valid_rng_state(seed: int = 42):
|
||||
@@ -423,6 +424,14 @@ class TestGeneratorStateValidation:
|
||||
with pytest.raises(ValueError, match="version"):
|
||||
GeneratorState.from_dict(data)
|
||||
|
||||
def test_from_dict_rejects_known_old_state_version_loudly(self):
|
||||
"""Известная старая версия state даёт отдельную ошибку миграции."""
|
||||
data = _make_valid_state_data()
|
||||
data["version"] = "2.0"
|
||||
|
||||
with pytest.raises(UnsupportedStateVersionError, match="2.0.*3.0"):
|
||||
GeneratorState.from_dict(data)
|
||||
|
||||
|
||||
class TestKafkaStateManager:
|
||||
"""Тесты менеджера состояния."""
|
||||
|
||||
Reference in New Issue
Block a user