fix(generator): исключено смешение миров при запуске

- Зачем:
  - старый state и ручной live-генератор могли тихо писать события в новый мир данных.
- Что:
  - добавлена общая проверка чистого стенда для Airflow и host-путей.
  - защищены backfill, import, continue и clean-start сценарии live-generator.
  - описан миграционный отказ старого state.
- Проверка:
  - make generator-test; docker compose config --quiet; py_compile; bash -n.

generator/src/clickstream_generator/airflow_control.py

@@ -25,6 +25,11 @@ from clickstream_generator.startup_history_artifact import (
     load_startup_history_artifact,
     validate_startup_history_artifact,
 )
+from clickstream_generator.stand_clean import (
+    STG_EMPTY_SQL,
+    assert_kafka_data_topics_empty,
+    assert_stg_counts_empty,
+)
 
 
 AIRFLOW_DATA_DIR = "/opt/airflow/data"
@@ -43,14 +48,6 @@ WORLD_OVERRIDE_KEYS = {
     "GEN_MAX_EVENTS_PER_TICK",
 }
 
-STG_EMPTY_SQL = """
-SELECT
-    (SELECT count() FROM stg.browser_raw) AS browser_raw,
-    (SELECT count() FROM stg.location_raw) AS location_raw,
-    (SELECT count() FROM stg.device_raw) AS device_raw,
-    (SELECT count() FROM stg.geo_raw) AS geo_raw
-"""
-
 CLICKHOUSE_STATS_SQL = """
 WITH
     toDateTime64('{model_t0}', 6) AS t0,
@@ -101,13 +98,7 @@ def build_control_env(
 
 def assert_stand_clean(kafka_bootstrap_servers: str, clickhouse_hook) -> None:
     """Проверяет, что backfill/import не смешает миры."""
-    try:
-        KafkaTopicInspector(kafka_bootstrap_servers).assert_data_topics_empty()
-    except RuntimeError as exc:
-        raise RuntimeError(
-            "Стенд не чистый: в Kafka data-топиках уже есть сообщения. "
-            "Выполните make clean с консоли и повторите операцию."
-        ) from exc
+    assert_kafka_data_topics_empty(kafka_bootstrap_servers)
     assert_stg_tables_empty(clickhouse_hook)
 
 
@@ -115,18 +106,7 @@ def assert_stg_tables_empty(clickhouse_hook) -> None:
     """Падает, если в STG уже есть строки."""
     result = clickhouse_hook.execute(STG_EMPTY_SQL)
     counts = result[0] if result else ()
-    names = ("stg.browser_raw", "stg.location_raw", "stg.device_raw", "stg.geo_raw")
-    dirty = [
-        f"{name}={int(count)}"
-        for name, count in zip(names, counts)
-        if int(count) > 0
-    ]
-    if dirty:
-        raise RuntimeError(
-            "Стенд не чистый: в STG уже есть строки ("
-            + ", ".join(dirty)
-            + "). Выполните make clean с консоли и повторите операцию."
-        )
+    assert_stg_counts_empty(counts)
 
 
 def assert_live_generator_not_running(url: str = GENERATOR_METRICS_URL) -> None:

generator/src/clickstream_generator/kafka_io.py

@@ -9,6 +9,7 @@ from datetime import datetime
 
 from clickstream_generator.metrics import METRICS_ERRORS_TOTAL, METRICS_EVENTS_TOTAL
 from clickstream_generator.state import GeneratorState
+from clickstream_generator.state import UnsupportedStateVersionError
 
 
 logger = logging.getLogger("generator")
@@ -178,14 +179,24 @@ class KafkaStateManager:
                     f"Restored state: tick={last_state.get('tick')}, "
                     f"last_batch_id={last_state.get('last_batch_id')}"
                 )
-                restored = GeneratorState.from_dict_safe(last_state)
-                if restored is None:
+                try:
+                    restored = GeneratorState.from_dict(last_state)
+                except UnsupportedStateVersionError:
+                    logger.error(
+                        "Unsupported generator state version %s",
+                        last_state.get("version", "1.0"),
+                    )
+                    raise
+                except ValueError:
                     logger.warning("State data was invalid, starting fresh")
+                    return None
                 return restored
 
             logger.info("No previous state found, starting fresh")
             return None
 
+        except UnsupportedStateVersionError:
+            raise
         except Exception as e:
             logger.warning(f"Failed to load state: {e}, starting fresh")
             return None

generator/src/clickstream_generator/service.py

@@ -31,6 +31,7 @@ from clickstream_generator.startup_history_artifact import (
     generation_settings_from_config,
     write_startup_history_artifact,
 )
+from clickstream_generator.state import UnsupportedStateVersionError
 
 
 logger = logging.getLogger("generator")
@@ -98,7 +99,21 @@ class GeneratorService:
                 return
 
             if not self.config.state_reset:
-                restored_state = self.state_manager.load()
+                try:
+                    restored_state = self.state_manager.load()
+                except UnsupportedStateVersionError as exc:
+                    logger.error(
+                        "Readable generator state uses an old format: version %s. "
+                        "Run make clean or set GEN_STATE_RESET=true only if you "
+                        "intentionally start a new world.",
+                        exc.found_version,
+                    )
+                    raise IncompatibleStateError(
+                        f"state version {exc.found_version} is not supported "
+                        f"by this code version ({exc.expected_version}); "
+                        "run make clean, or set GEN_STATE_RESET=true to start "
+                        "a new world intentionally"
+                    ) from exc
                 if restored_state:
                     try:
                         self.manifest_manager = KafkaStartupHistoryManifest(

generator/src/clickstream_generator/stand_clean.py

@@ -0,0 +1,168 @@
+"""Общая предпроверка чистого стенда перед новым миром."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import urllib.error
+import urllib.request
+from collections.abc import Iterable
+
+from clickstream_generator.config import Config
+from clickstream_generator.kafka_io import (
+    KafkaStateManager,
+    KafkaStartupHistoryManifest,
+)
+from clickstream_generator.service import GeneratorService, IncompatibleStateError
+from clickstream_generator.state import UnsupportedStateVersionError
+from clickstream_generator.startup_history_artifact import KafkaTopicInspector
+
+
+STG_EMPTY_SQL = """
+SELECT
+    (SELECT count() FROM stg.browser_raw) AS browser_raw,
+    (SELECT count() FROM stg.location_raw) AS location_raw,
+    (SELECT count() FROM stg.device_raw) AS device_raw,
+    (SELECT count() FROM stg.geo_raw) AS geo_raw
+"""
+
+STG_TABLE_NAMES = (
+    "stg.browser_raw",
+    "stg.location_raw",
+    "stg.device_raw",
+    "stg.geo_raw",
+)
+
+
+def assert_kafka_data_topics_empty(kafka_bootstrap_servers: str) -> None:
+    """Падает, если в Kafka data-топиках уже есть сообщения."""
+    try:
+        KafkaTopicInspector(kafka_bootstrap_servers).assert_data_topics_empty()
+    except RuntimeError as exc:
+        raise RuntimeError(
+            "Стенд не чистый: в Kafka data-топиках уже есть сообщения. "
+            "Выполните make clean и повторите операцию."
+        ) from exc
+
+
+def assert_stg_counts_empty(counts: Iterable[int]) -> None:
+    """Падает, если в STG уже есть строки."""
+    dirty = [
+        f"{name}={int(count)}"
+        for name, count in zip(STG_TABLE_NAMES, counts)
+        if int(count) > 0
+    ]
+    if dirty:
+        raise RuntimeError(
+            "Стенд не чистый: в STG уже есть строки ("
+            + ", ".join(dirty)
+            + "). Выполните make clean и повторите операцию."
+        )
+
+
+def parse_stg_counts(value: str) -> tuple[int, int, int, int]:
+    """Разбирает TSV-строку ClickHouse с четырьмя счётчиками STG."""
+    parts = value.strip().split()
+    if len(parts) != len(STG_TABLE_NAMES):
+        raise RuntimeError(
+            "Не удалось проверить чистоту STG: ClickHouse вернул "
+            f"{len(parts)} значений вместо {len(STG_TABLE_NAMES)}."
+        )
+    return tuple(int(part) for part in parts)  # type: ignore[return-value]
+
+
+def assert_stand_clean(kafka_bootstrap_servers: str, stg_counts: Iterable[int]) -> None:
+    """Проверяет Kafka data-топики и STG перед backfill/import."""
+    assert_kafka_data_topics_empty(kafka_bootstrap_servers)
+    assert_stg_counts_empty(stg_counts)
+
+
+def assert_live_generator_not_running(live_metrics_url: str) -> None:
+    """Падает, если live-генератор отвечает на metrics-порту."""
+    try:
+        urllib.request.urlopen(live_metrics_url, timeout=2).close()
+    except (urllib.error.URLError, TimeoutError, OSError):
+        return
+    raise RuntimeError(
+        "Live-генератор уже запущен. Остановите его через make generator-down "
+        "или make clean перед новым backfill/import."
+    )
+
+
+def assert_continue_has_state_or_clean_stand(
+    kafka_bootstrap_servers: str,
+    stg_counts: Iterable[int],
+    config: Config,
+) -> None:
+    """Разрешает continue только от совместимого state или чистого стенда."""
+    state_manager = KafkaStateManager(kafka_bootstrap_servers)
+    try:
+        state = state_manager.load()
+    except UnsupportedStateVersionError as exc:
+        raise RuntimeError(
+            f"State записан старой версией {exc.found_version}. "
+            "Выполните make clean или явно начните новый мир через make generator-reset."
+        ) from exc
+    finally:
+        state_manager.close()
+
+    if state is None:
+        assert_stand_clean(kafka_bootstrap_servers, stg_counts)
+        return
+
+    service = GeneratorService(config)
+    manifest_manager = KafkaStartupHistoryManifest(kafka_bootstrap_servers)
+    try:
+        manifest = manifest_manager.load()
+    finally:
+        manifest_manager.close()
+
+    if service._is_startup_history_state(state, manifest):
+        return
+    if service._is_startup_history_marker(state):
+        mismatches = service._startup_history_mismatch_fields(state, manifest)
+        raise RuntimeError(
+            "State стартовой истории несовместим с текущими настройками: "
+            + ", ".join(mismatches)
+            + ". Выполните make clean или используйте тот же профиль."
+        )
+    try:
+        service._validate_live_state_config(state)
+    except IncompatibleStateError as exc:
+        raise RuntimeError(str(exc)) from exc
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Проверить, что стенд чист перед новым миром генератора."
+    )
+    parser.add_argument("--mode", choices=("clean", "continue", "live"), default="clean")
+    parser.add_argument("--kafka-bootstrap-servers")
+    parser.add_argument("--stg-counts")
+    parser.add_argument("--live-metrics-url")
+    args = parser.parse_args(argv)
+
+    try:
+        if args.live_metrics_url:
+            assert_live_generator_not_running(args.live_metrics_url)
+        if args.mode == "live":
+            return 0
+        if not args.kafka_bootstrap_servers or not args.stg_counts:
+            raise RuntimeError("Нужны --kafka-bootstrap-servers и --stg-counts.")
+        stg_counts = parse_stg_counts(args.stg_counts)
+        if args.mode == "continue":
+            assert_continue_has_state_or_clean_stand(
+                args.kafka_bootstrap_servers,
+                stg_counts,
+                Config(),
+            )
+        else:
+            assert_stand_clean(args.kafka_bootstrap_servers, stg_counts)
+    except RuntimeError as exc:
+        print(f"Ошибка: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

generator/src/clickstream_generator/state.py

@@ -11,6 +11,19 @@ logger = logging.getLogger("generator")
 
 
 STATE_VERSION = "3.0"
+KNOWN_OLD_STATE_VERSIONS = {"2.0"}
+
+
+class UnsupportedStateVersionError(ValueError):
+    """State записан в старом формате и требует явного решения оператора."""
+
+    def __init__(self, found_version: str, expected_version: str = STATE_VERSION):
+        self.found_version = found_version
+        self.expected_version = expected_version
+        super().__init__(
+            f"unsupported state version {found_version}; "
+            f"current version is {expected_version}"
+        )
 
 
 def _nested_list_to_tuple(obj):
@@ -234,6 +247,8 @@ class GeneratorState:
                 raise ValueError("state must be an object")
             version = data.get("version", "1.0")
             if version != STATE_VERSION:
+                if str(version) in KNOWN_OLD_STATE_VERSIONS:
+                    raise UnsupportedStateVersionError(str(version))
                 logger.warning(
                     "Unsupported generator state version %s, will start fresh",
                     version,
@@ -283,6 +298,8 @@ class GeneratorState:
                 active_visits=data.get("active_visits", []),
                 pending_visit_births=data.get("pending_visit_births", 0.0),
             )
+        except UnsupportedStateVersionError:
+            raise
         except Exception as e:
             logger.warning(f"Invalid state format, will start fresh: {e}")
             raise ValueError(f"Invalid state: {e}")