feat(generator): добавлен артефакт стартовой истории

- Зачем:
  - чистый стенд должен восстанавливать стартовую историю без повторной генерации.
- Что:
  - добавлены export/import артефакта через Kafka и compact-топики.
  - добавлена manifest-aware сверка ClickHouse и защита от смешения state.
  - добавлен runbook использования стартовой истории.
- Проверка:
  - uv run --with-requirements generator/requirements.txt pytest generator/tests -q.
  - bash -n scripts/check_startup_history_manifest.sh scripts/export_startup_history_artifact.sh scripts/import_startup_history_artifact.sh.
  - git diff --check.

generator/src/clickstream_generator/service.py

@@ -1,7 +1,5 @@
 """Основной сервисный цикл генератора."""
 
-import hashlib
-import json
 import logging
 import sys
 import time
@@ -27,108 +25,19 @@ from clickstream_generator.metrics import (
     METRICS_TICK_DURATION,
 )
 from clickstream_generator.runtime import TickStreamGenerator
+from clickstream_generator.startup_history_artifact import (
+    StartupHistoryArtifactBuilder,
+    build_manifest,
+    generation_settings_from_config,
+    write_startup_history_artifact,
+)
 
 
 logger = logging.getLogger("generator")
 
 
-class _TopicManifestStats:
-    """Накопительные счётчики одного Kafka-топика для manifest."""
-
-    def __init__(self):
-        self.rows = 0
-        self.min_event_timestamp: str | None = None
-        self.max_event_timestamp: str | None = None
-        self._checksum = hashlib.sha256()
-
-    @property
-    def checksum(self) -> str:
-        return self._checksum.hexdigest()
-
-    def add(self, event: dict, event_timestamp: str | None = None) -> None:
-        self.rows += 1
-        self._checksum.update(
-            json.dumps(event, sort_keys=True, ensure_ascii=True).encode("utf-8")
-        )
-        timestamp = event_timestamp if event_timestamp is not None else event.get("event_timestamp")
-        self.add_timestamp(timestamp)
-
-    def add_timestamp(self, timestamp: str | None) -> None:
-        if timestamp is None:
-            return
-        if self.min_event_timestamp is None or timestamp < self.min_event_timestamp:
-            self.min_event_timestamp = timestamp
-        if self.max_event_timestamp is None or timestamp > self.max_event_timestamp:
-            self.max_event_timestamp = timestamp
-
-    def to_dict(self) -> dict:
-        return {
-            "rows": self.rows,
-            "min_event_timestamp": self.min_event_timestamp,
-            "max_event_timestamp": self.max_event_timestamp,
-            "checksum_sha256": self.checksum,
-        }
-
-
-class _ManifestCounters:
-    """Счётчики стартовой истории для manifest."""
-
-    def __init__(self):
-        self.topic_stats = {
-            topic: _TopicManifestStats()
-            for topic in ("browser_events", "location_events", "device_events", "geo_events")
-        }
-        self.click_ids: set[str] = set()
-        self.user_ids: set[str] = set()
-
-    def add_batch(self, batch: dict[str, list[dict]]) -> None:
-        browser_events = batch.get("browser_events", [])
-        event_timestamps = {
-            event["event_id"]: event.get("event_timestamp")
-            for event in browser_events
-            if event.get("event_id") and event.get("event_timestamp")
-        }
-        click_timestamps: dict[str, list[str]] = {}
-        for event in browser_events:
-            click_id = event.get("click_id")
-            timestamp = event.get("event_timestamp")
-            if click_id and timestamp:
-                click_timestamps.setdefault(click_id, []).append(timestamp)
-
-        for topic, events in batch.items():
-            stats = self.topic_stats[topic]
-            for event in events:
-                event_timestamp = event.get("event_timestamp")
-                if event_timestamp is None and topic == "location_events":
-                    event_timestamp = event_timestamps.get(event.get("event_id"))
-                if event_timestamp is None and topic in ("device_events", "geo_events"):
-                    timestamps = click_timestamps.get(event.get("click_id"))
-                    if timestamps:
-                        event_timestamp = min(timestamps)
-                        stats.add_timestamp(max(timestamps))
-                stats.add(event, event_timestamp=event_timestamp)
-                click_id = event.get("click_id")
-                if topic == "browser_events" and click_id:
-                    self.click_ids.add(click_id)
-                user_id = event.get("user_domain_id")
-                if topic == "device_events" and user_id:
-                    self.user_ids.add(user_id)
-
-    def to_manifest_topics(self) -> dict:
-        return {
-            topic: stats.to_dict()
-            for topic, stats in self.topic_stats.items()
-        }
-
-    def to_manifest_totals(self) -> dict:
-        browser_stats = self.topic_stats["browser_events"]
-        return {
-            "events": browser_stats.rows,
-            "visits": len(self.click_ids),
-            "users": len(self.user_ids),
-            "min_event_timestamp": browser_stats.min_event_timestamp,
-            "max_event_timestamp": browser_stats.max_event_timestamp,
-        }
+class IncompatibleStateError(ValueError):
+    """Читаемый state относится к другому миру генерации."""
 
 
 class GeneratorService:
@@ -201,8 +110,15 @@ class GeneratorService:
                                 model_t_end=model_t_end,
                             )
                         elif self._is_startup_history_marker(restored_state):
-                            raise ValueError(
-                                "startup-history state without matching manifest"
+                            mismatches = self._startup_history_mismatch_fields(
+                                restored_state,
+                                manifest,
+                            )
+                            raise IncompatibleStateError(
+                                "startup-history state mismatch: "
+                                + ", ".join(mismatches)
+                                + "; set GEN_STATE_RESET=true to start a new "
+                                "world intentionally"
                             )
                         else:
                             self._restore_live_state(
@@ -214,6 +130,13 @@ class GeneratorService:
                             f"model_time={self._model_time.isoformat()}, "
                             f"last_batch_id={restored_state.last_batch_id}"
                         )
+                    except IncompatibleStateError:
+                        logger.error(
+                            "Readable generator state is incompatible with "
+                            "current settings; set GEN_STATE_RESET=true to "
+                            "start a new world intentionally."
+                        )
+                        raise
                     except Exception as e:
                         logger.warning(
                             f"State data was invalid, starting fresh: {e}"
@@ -290,27 +213,36 @@ class GeneratorService:
         """Проверяет, что state относится к текущей конфигурации live-запуска."""
         mismatches = []
         if state.gen_seed != self.config.seed:
-            mismatches.append("gen_seed")
+            mismatches.append("GEN_SEED")
         if self._as_aware_utc(state.model_t0) != self.config.model_t0:
-            mismatches.append("model_t0")
+            mismatches.append("GEN_MODEL_T0")
         if state.model_timezone != self.config.model_timezone:
-            mismatches.append("model_timezone")
+            mismatches.append("GEN_MODEL_TIMEZONE")
         if abs(state.model_time_speed - self.config.model_time_speed) > 1e-9:
-            mismatches.append("model_time_speed")
+            mismatches.append("GEN_MODEL_TIME_SPEED")
 
         if mismatches:
-            raise ValueError(
-                "state config mismatch: " + ", ".join(mismatches)
+            raise IncompatibleStateError(
+                "state config mismatch: "
+                + ", ".join(mismatches)
+                + "; set GEN_STATE_RESET=true to start a new world intentionally"
             )
 
     def _is_startup_history_state(self, state, manifest: dict | None) -> bool:
         """Проверяет, что state совпадает со слепком стартовой истории."""
+        return not self._startup_history_mismatch_fields(state, manifest)
+
+    def _startup_history_mismatch_fields(self, state, manifest: dict | None) -> list[str]:
+        """Возвращает поля, по которым startup-history state не совпал."""
+        mismatches = []
         if not manifest or manifest.get("run_mode") != "backfill":
-            return False
+            return ["generator_startup_history_manifest"]
 
         expected_state = manifest.get("state") or {}
+        if manifest.get("state_version") != state.version:
+            mismatches.append("state_version")
         if expected_state.get("last_batch_id") != state.last_batch_id:
-            return False
+            mismatches.append("state.last_batch_id")
 
         try:
             model_t_end = self._as_aware_utc(
@@ -320,32 +252,34 @@ class GeneratorService:
                 datetime.fromisoformat(manifest["model_t0"])
             )
         except (KeyError, TypeError, ValueError):
-            return False
+            return ["manifest.model_t0", "manifest.model_t_end"]
 
         if self._as_aware_utc(state.model_timestamp) != model_t_end:
-            return False
+            mismatches.append("GEN_MODEL_T_END")
         if self._as_aware_utc(state.model_t0) != model_t0:
-            return False
+            mismatches.append("GEN_MODEL_T0")
         if state.gen_seed != manifest.get("gen_seed"):
-            return False
+            mismatches.append("GEN_SEED")
         if state.model_timezone != manifest.get("model_timezone"):
-            return False
+            mismatches.append("GEN_MODEL_TIMEZONE")
         if (
             self.config.model_t_end is not None
             and self.config.model_t_end != model_t_end
         ):
-            return False
+            mismatches.append("GEN_MODEL_T_END")
         if model_t0 != self.config.model_t0:
-            return False
+            mismatches.append("GEN_MODEL_T0")
         if manifest.get("gen_seed") != self.config.seed:
-            return False
+            mismatches.append("GEN_SEED")
         if manifest.get("model_timezone") != self.config.model_timezone:
-            return False
+            mismatches.append("GEN_MODEL_TIMEZONE")
 
         settings = manifest.get("generation_settings") or {}
         if abs(state.model_time_speed - settings.get("model_time_speed", -1)) > 1e-9:
-            return False
-        return self._generation_settings() == settings
+            mismatches.append("GEN_MODEL_TIME_SPEED")
+        if self._generation_settings() != settings:
+            mismatches.append("generation_settings")
+        return list(dict.fromkeys(mismatches))
 
     def _is_startup_history_marker(self, state) -> bool:
         """Отличает state стартовой истории от обычного live-state."""
@@ -400,7 +334,7 @@ class GeneratorService:
             self.config.model_t0.isoformat(),
             self.config.model_t_end.isoformat(),
         )
-        counters = _ManifestCounters()
+        artifact_builder = StartupHistoryArtifactBuilder()
 
         while self._model_time < self.config.model_t_end:
             self._tick += 1
@@ -415,7 +349,7 @@ class GeneratorService:
             )
             total_sent, sent_counts, status = self._publish_batch(batch)
             self._raise_on_backfill_publish_error(batch_id, status, sent_counts)
-            counters.add_batch(batch)
+            artifact_builder.add_batch(batch)
             self._write_batch_history(
                 batch_id=batch_id,
                 started_at=started_at,
@@ -441,7 +375,7 @@ class GeneratorService:
                 final_status,
                 sent_counts,
             )
-            counters.add_batch(final_batch)
+            artifact_builder.add_batch(final_batch)
             self._write_batch_history(
                 batch_id=batch_id,
                 started_at=started_at,
@@ -455,7 +389,7 @@ class GeneratorService:
             self.publisher.flush()
 
         self._model_time = self.config.model_t_end
-        state_batch_id = self._startup_state_batch_id(counters)
+        state_batch_id = self._startup_state_batch_id(artifact_builder.counters)
         state = self.stream.to_state(
             tick=self._tick,
             rng_state=self.generator.rng.getstate(),
@@ -469,8 +403,9 @@ class GeneratorService:
             gen_seed=self.config.seed,
         )
 
-        manifest = self._build_startup_history_manifest(
-            counters=counters,
+        manifest = build_manifest(
+            config=self.config,
+            counters=artifact_builder.counters,
             state=state,
         )
         if self.manifest_manager:
@@ -481,6 +416,17 @@ class GeneratorService:
             self.state_manager.save(state)
             self.state_manager.flush()
 
+        if self.config.startup_history_artifact is not None:
+            artifact = artifact_builder.to_artifact(manifest=manifest, state=state)
+            write_startup_history_artifact(
+                self.config.startup_history_artifact,
+                artifact,
+            )
+            logger.info(
+                "Startup history artifact written: %s",
+                self.config.startup_history_artifact,
+            )
+
         logger.info(
             "Backfill completed: events=%s, visits=%s, users=%s, final_sent=%s",
             manifest["totals"]["events"],
@@ -565,40 +511,10 @@ class GeneratorService:
         return f"startup-history-{digest}"
 
     def _build_startup_history_manifest(self, counters, state) -> dict:
-        return {
-            "manifest_version": "1.0",
-            "generated_at": datetime.now(timezone.utc).isoformat(),
-            "gen_seed": self.config.seed,
-            "model_t0": self.config.model_t0.isoformat(),
-            "model_t_end": self.config.model_t_end.isoformat(),
-            "model_timezone": self.config.model_timezone,
-            "run_mode": "backfill",
-            "generation_settings": self._generation_settings(),
-            "state_version": state.version,
-            "state": {
-                "topic": KafkaStateManager.STATE_TOPIC,
-                "key": KafkaStateManager.STATE_KEY,
-                "last_batch_id": state.last_batch_id,
-                "model_timestamp": state.model_timestamp.isoformat(),
-            },
-            "topics": counters.to_manifest_topics(),
-            "totals": counters.to_manifest_totals(),
-        }
+        return build_manifest(config=self.config, counters=counters, state=state)
 
     def _generation_settings(self) -> dict:
-        return {
-            "tick_seconds": self.config.tick_seconds,
-            "lambda_base_per_min": self.config.lambda_base_per_min,
-            "jitter_pct": self.config.jitter_pct,
-            "min_events_per_tick": self.config.min_events_per_tick,
-            "max_events_per_tick": self.config.max_events_per_tick,
-            "max_session_events": self.config.max_session_events,
-            "max_active_sessions": self.config.max_active_sessions,
-            "population_max": self.config.population_max,
-            "p_new_user": self.config.p_new_user,
-            "min_return_minutes": self.config.min_return_minutes,
-            "model_time_speed": self.config.model_time_speed,
-        }
+        return generation_settings_from_config(self.config)
 
     def _main_loop(self):
         """Основной цикл тиков."""