feat(generator): добавлено состояние версии 2 для рестартов

- Зачем:
  - генератор должен переживать рестарт без потери популяции пользователей и коротко прерванных активных визитов.
- Что:
  - добавлен компактный state v2 для популяции, активных визитов и остатка бюджета рождений.
  - сервис генератора переведён на единый тиковый поток с сохранением и восстановлением состояния.
  - добавлена безопасная деградация для старого state v1 и битого state v2.
  - покрыты короткий и долгий простой, reset состояния и валидация вложенного state.
- Проверка:
  - uv run --with-requirements generator/requirements.txt pytest generator/tests -q.
  - git diff --check.

generator/src/clickstream_generator/runtime.py

@@ -1,13 +1,16 @@
 """Тиковый слой генератора с активными визитами между вызовами."""
 
+import uuid
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from weakref import WeakKeyDictionary
 
 from clickstream_generator.generation import EXPECTED_VISIT_EVENTS, EventGenerator
+from clickstream_generator.state import GeneratorState
 
 
 TOPICS = ("browser_events", "location_events", "device_events", "geo_events")
+RESTART_VISIT_GRACE = timedelta(minutes=30)
 
 
 @dataclass
@@ -122,6 +125,22 @@ def _parse_timestamp(value: str) -> datetime:
     return datetime.fromisoformat(value.replace(" ", "T"))
 
 
+def _datetime_to_state(value: datetime | None) -> str | None:
+    return value.isoformat() if value is not None else None
+
+
+def _timestamp_to_state_offset(started_at: datetime, timestamp: datetime) -> int:
+    return int((timestamp - started_at).total_seconds() * 1_000_000)
+
+
+def _format_event_timestamp(timestamp: datetime) -> str:
+    return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
+
+
+def _stable_event_id(click_id: str, event_index: int) -> str:
+    return str(uuid.uuid5(uuid.NAMESPACE_URL, f"{click_id}:{event_index}"))
+
+
 def _normalize_tick_time(tick_started_at: datetime | None) -> datetime:
     tick_time = tick_started_at or datetime.now(timezone.utc)
     if tick_time.tzinfo is not None:
@@ -146,6 +165,205 @@ class TickStreamGenerator:
     def population_user_ids(self) -> set[str]:
         return {user.user_domain_id for user in self.population.users}
 
+    @property
+    def active_visit_count(self) -> int:
+        return len(self.active_visits)
+
+    def to_state(
+        self,
+        tick: int,
+        rng_state: tuple,
+        last_batch_id: str,
+        last_timestamp: datetime,
+    ) -> GeneratorState:
+        """Возвращает JSON-сериализуемый снимок тикового слоя."""
+        return GeneratorState(
+            tick=tick,
+            rng_state=rng_state,
+            last_batch_id=last_batch_id,
+            last_timestamp=last_timestamp,
+            population=[
+                {
+                    "user_domain_id": user.user_domain_id,
+                    "seed_click_id": user.seed_click_id,
+                    "active_click_id": user.active_click_id,
+                    "last_finished_at": _datetime_to_state(user.last_finished_at),
+                }
+                for user in self.population.users
+            ],
+            active_visits=[
+                self._visit_to_state(visit)
+                for visit in self.active_visits
+            ],
+            pending_visit_births=self._pending_visit_births,
+        )
+
+    def _visit_to_state(self, visit: ActiveVisit) -> dict:
+        started_at = visit.timestamps[0]
+        browser_events = visit.batch["browser_events"]
+        location_events = visit.batch["location_events"]
+        return {
+            "user_domain_id": visit.user.user_domain_id if visit.user else None,
+            "click_id": browser_events[0]["click_id"],
+            "next_index": visit.next_index,
+            "started_at": started_at.isoformat(),
+            "offsets_us": [
+                _timestamp_to_state_offset(started_at, timestamp)
+                for timestamp in visit.timestamps
+            ],
+            "page_url_paths": [
+                event["page_url_path"]
+                for event in location_events
+            ],
+        }
+
+    def restore_state(
+        self,
+        state: GeneratorState,
+        restarted_at: datetime | None = None,
+    ) -> None:
+        """Восстанавливает популяцию и активные визиты из state v2."""
+        users = [
+            self._user_from_state(item)
+            for item in state.population
+        ]
+        users_by_id = {user.user_domain_id: user for user in users}
+
+        self.population.users = users
+        self.active_visits = []
+        restarted_time = (
+            _normalize_tick_time(restarted_at)
+            if restarted_at is not None
+            else None
+        )
+        for item in state.active_visits:
+            visit = self._visit_from_state(item, users_by_id)
+            if self._is_overdue_after_restart(visit, restarted_time):
+                self.population.finish_visit(
+                    visit.user,
+                    self._last_released_at(visit, state.last_timestamp),
+                )
+                continue
+            self.active_visits.append(visit)
+        self._pending_visit_births = state.pending_visit_births
+
+    def _user_from_state(self, item: dict) -> UserProfile:
+        seed_click_id = item["seed_click_id"]
+        if seed_click_id not in self.generator.dictionary.device_by_click_id:
+            raise ValueError(f"Unknown user seed_click_id: {seed_click_id}")
+        if seed_click_id not in self.generator.dictionary.geo_by_click_id:
+            raise ValueError(f"Unknown user geo seed_click_id: {seed_click_id}")
+
+        user_domain_id = item["user_domain_id"]
+        device = {
+            **self.generator.dictionary.device_by_click_id[seed_click_id],
+            "user_domain_id": user_domain_id,
+        }
+        geo = self.generator.dictionary.geo_by_click_id[seed_click_id]
+        return UserProfile(
+            user_domain_id=user_domain_id,
+            seed_click_id=seed_click_id,
+            device=device,
+            geo=geo,
+            active_click_id=item.get("active_click_id"),
+            last_finished_at=(
+                _parse_timestamp(item["last_finished_at"])
+                if item.get("last_finished_at")
+                else None
+            ),
+        )
+
+    def _visit_from_state(
+        self,
+        item: dict,
+        users_by_id: dict[str, UserProfile],
+    ) -> ActiveVisit:
+        user = users_by_id.get(item["user_domain_id"])
+        if user is None:
+            raise ValueError(f"Unknown active visit user: {item['user_domain_id']}")
+
+        started_at = _parse_timestamp(item["started_at"])
+        timestamps = [
+            started_at + timedelta(microseconds=offset_us)
+            for offset_us in item["offsets_us"]
+        ]
+        batch = self._compact_visit_batch(
+            click_id=item["click_id"],
+            user=user,
+            timestamps=timestamps,
+            page_url_paths=item["page_url_paths"],
+        )
+        return ActiveVisit(
+            batch=batch,
+            timestamps=timestamps,
+            user=user,
+            next_index=item["next_index"],
+        )
+
+    def _compact_visit_batch(
+        self,
+        click_id: str,
+        user: UserProfile,
+        timestamps: list[datetime],
+        page_url_paths: list[str],
+    ) -> dict[str, list[dict]]:
+        batch = _empty_batch()
+        browser_templates = self.generator.dictionary.browser_by_click_id.get(
+            user.seed_click_id,
+            self.generator.dictionary.browser_events,
+        )
+
+        for event_index, (timestamp, page_url_path) in enumerate(
+            zip(timestamps, page_url_paths)
+        ):
+            browser_template = browser_templates[event_index % len(browser_templates)]
+            location_template = self.generator.dictionary.location_by_event_id.get(
+                browser_template["event_id"],
+                self.generator.dictionary.location_events[0],
+            )
+            event_id = _stable_event_id(click_id, event_index)
+            batch["browser_events"].append(
+                {
+                    **browser_template,
+                    "event_id": event_id,
+                    "click_id": click_id,
+                    "event_timestamp": _format_event_timestamp(timestamp),
+                }
+            )
+            batch["location_events"].append(
+                {
+                    **location_template,
+                    "event_id": event_id,
+                    "page_url": f"http://www.dummywebsite.com{page_url_path}",
+                    "page_url_path": page_url_path,
+                }
+            )
+            batch["device_events"].append({**user.device, "click_id": click_id})
+            batch["geo_events"].append({**user.geo, "click_id": click_id})
+
+        return batch
+
+    def _last_released_at(
+        self,
+        visit: ActiveVisit,
+        fallback: datetime,
+    ) -> datetime:
+        if visit.next_index > 0:
+            return visit.timestamps[visit.next_index - 1]
+        if fallback.tzinfo is not None:
+            return fallback.astimezone(timezone.utc).replace(tzinfo=None)
+        return fallback
+
+    def _is_overdue_after_restart(
+        self,
+        visit: ActiveVisit,
+        restarted_time: datetime | None,
+    ) -> bool:
+        if restarted_time is None or visit.is_finished:
+            return False
+        next_timestamp = visit.timestamps[visit.next_index]
+        return restarted_time - next_timestamp > RESTART_VISIT_GRACE
+
     def generate_tick(
         self,
         event_budget: int,

generator/src/clickstream_generator/service.py

@@ -23,8 +23,7 @@ from clickstream_generator.metrics import (
     METRICS_LAST_SUCCESS,
     METRICS_TICK_DURATION,
 )
-from clickstream_generator.runtime import generate_tick_batch
-from clickstream_generator.state import GeneratorState
+from clickstream_generator.runtime import TickStreamGenerator
 
 
 logger = logging.getLogger("generator")
@@ -37,6 +36,7 @@ class GeneratorService:
         self.config = config
         self.dictionary = EventDictionary.load(config.data_dir)
         self.generator = EventGenerator(self.dictionary, config)
+        self.stream = TickStreamGenerator(self.generator)
         self.publisher: KafkaPublisher | None = None
         self.history: KafkaBatchHistory | None = None
         self.state_manager: KafkaStateManager | None = None
@@ -71,12 +71,24 @@ class GeneratorService:
             if not self.config.state_reset:
                 restored_state = self.state_manager.load()
                 if restored_state:
-                    self._tick = restored_state.tick
-                    self.generator.rng.setstate(restored_state.rng_state)
-                    logger.info(
-                        f"Restored state: continuing from tick {self._tick}, "
-                        f"last_batch_id={restored_state.last_batch_id}"
-                    )
+                    try:
+                        self.generator.rng.setstate(restored_state.rng_state)
+                        self.stream.restore_state(
+                            restored_state,
+                            restarted_at=datetime.now(timezone.utc),
+                        )
+                        self._tick = restored_state.tick
+                        logger.info(
+                            f"Restored state: continuing from tick {self._tick}, "
+                            f"last_batch_id={restored_state.last_batch_id}"
+                        )
+                    except Exception as e:
+                        logger.warning(
+                            f"State data was invalid, starting fresh: {e}"
+                        )
+                        self.generator = EventGenerator(self.dictionary, self.config)
+                        self.stream = TickStreamGenerator(self.generator)
+                        self._tick = 0
             else:
                 logger.info("State reset requested, starting fresh")
         else:
@@ -108,7 +120,7 @@ class GeneratorService:
             return
 
         try:
-            state = GeneratorState(
+            state = self.stream.to_state(
                 tick=self._tick,
                 rng_state=self.generator.rng.getstate(),
                 last_batch_id=batch_id,
@@ -136,7 +148,7 @@ class GeneratorService:
                     logger.info(f"Generating with event budget ~{events_count}")
 
                     gen_start = time.time()
-                    batch = generate_tick_batch(self.generator, events_count)
+                    batch = self.stream.generate_tick(events_count)
                     gen_duration = time.time() - gen_start
 
                     pub_start = time.time()

generator/src/clickstream_generator/state.py

@@ -2,13 +2,16 @@
 
 import logging
 import random
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 
 
 logger = logging.getLogger("generator")
 
 
+STATE_VERSION = "2.0"
+
+
 def _nested_list_to_tuple(obj):
     """Рекурсивно преобразует list в tuple для восстановления RNG state."""
     if isinstance(obj, list):
@@ -16,6 +19,107 @@ def _nested_list_to_tuple(obj):
     return obj
 
 
+def _require_keys(item: dict, keys: tuple[str, ...], label: str) -> None:
+    missing = [key for key in keys if key not in item]
+    if missing:
+        raise ValueError(f"{label} missing fields: {', '.join(missing)}")
+
+
+def _validate_v2_payload(data: dict) -> None:
+    population = data.get("population")
+    active_visits = data.get("active_visits")
+    pending_visit_births = data.get("pending_visit_births", 0.0)
+
+    if not isinstance(population, list):
+        raise ValueError("population must be a list")
+    if not population:
+        raise ValueError("population must be non-empty")
+    if not isinstance(active_visits, list):
+        raise ValueError("active_visits must be a list")
+    if not isinstance(pending_visit_births, int | float):
+        raise ValueError("pending_visit_births must be a number")
+    if not 0 <= pending_visit_births < 1_000_000:
+        raise ValueError("pending_visit_births is out of range")
+
+    users_by_id = {}
+    for index, user in enumerate(population):
+        if not isinstance(user, dict):
+            raise ValueError(f"population[{index}] must be an object")
+        _require_keys(user, ("user_domain_id", "seed_click_id"), f"population[{index}]")
+        user_domain_id = user["user_domain_id"]
+        seed_click_id = user["seed_click_id"]
+        active_click_id = user.get("active_click_id")
+        last_finished_at = user.get("last_finished_at")
+        if not isinstance(user_domain_id, str) or not user_domain_id:
+            raise ValueError(f"population[{index}].user_domain_id must be a string")
+        if not isinstance(seed_click_id, str) or not seed_click_id:
+            raise ValueError(f"population[{index}].seed_click_id must be a string")
+        if active_click_id is not None and not isinstance(active_click_id, str):
+            raise ValueError(f"population[{index}].active_click_id must be a string or null")
+        if last_finished_at is not None:
+            if not isinstance(last_finished_at, str):
+                raise ValueError(f"population[{index}].last_finished_at must be a string or null")
+            datetime.fromisoformat(last_finished_at)
+        if user_domain_id in users_by_id:
+            raise ValueError(f"duplicate population user_domain_id: {user_domain_id}")
+        users_by_id[user_domain_id] = user
+
+    active_visit_pairs = set()
+    for index, visit in enumerate(active_visits):
+        if not isinstance(visit, dict):
+            raise ValueError(f"active_visits[{index}] must be an object")
+        _require_keys(
+            visit,
+            (
+                "user_domain_id",
+                "click_id",
+                "next_index",
+                "started_at",
+                "offsets_us",
+                "page_url_paths",
+            ),
+            f"active_visits[{index}]",
+        )
+        user_domain_id = visit["user_domain_id"]
+        click_id = visit["click_id"]
+        offsets = visit["offsets_us"]
+        page_url_paths = visit["page_url_paths"]
+        next_index = visit["next_index"]
+        if not isinstance(user_domain_id, str) or user_domain_id not in users_by_id:
+            raise ValueError(f"active_visits[{index}].user_domain_id is unknown")
+        if not isinstance(click_id, str) or not click_id:
+            raise ValueError(f"active_visits[{index}].click_id must be a string")
+        if not isinstance(offsets, list) or not offsets:
+            raise ValueError(f"active_visits[{index}].offsets_us must be a non-empty list")
+        if not all(isinstance(offset, int) and offset >= 0 for offset in offsets):
+            raise ValueError(f"active_visits[{index}].offsets_us must contain non-negative integers")
+        if offsets != sorted(offsets):
+            raise ValueError(f"active_visits[{index}].offsets_us must be sorted")
+        if not isinstance(page_url_paths, list) or len(page_url_paths) != len(offsets):
+            raise ValueError(
+                f"active_visits[{index}].page_url_paths must match offsets_us length"
+            )
+        if not all(isinstance(path, str) and path.startswith("/") for path in page_url_paths):
+            raise ValueError(f"active_visits[{index}].page_url_paths must contain paths")
+        if not isinstance(next_index, int) or not 0 <= next_index <= len(offsets):
+            raise ValueError(f"active_visits[{index}].next_index is out of range")
+        datetime.fromisoformat(visit["started_at"])
+
+        user_active_click_id = users_by_id[user_domain_id].get("active_click_id")
+        if user_active_click_id != click_id:
+            raise ValueError(
+                f"population active_click_id conflicts with active_visits[{index}]"
+            )
+        active_visit_pairs.add((user_domain_id, click_id))
+
+    for user_domain_id, user in users_by_id.items():
+        active_click_id = user.get("active_click_id")
+        if active_click_id and (user_domain_id, active_click_id) not in active_visit_pairs:
+            raise ValueError(
+                f"population user {user_domain_id} has active_click_id without active visit"
+            )
+
+
 @dataclass
 class GeneratorState:
     """Состояние генератора для восстановления после рестарта."""
@@ -24,7 +128,10 @@ class GeneratorState:
     rng_state: tuple
     last_batch_id: str
     last_timestamp: datetime
-    version: str = "1.0"
+    version: str = STATE_VERSION
+    population: list[dict] = field(default_factory=list)
+    active_visits: list[dict] = field(default_factory=list)
+    pending_visit_births: float = 0.0
 
     def to_dict(self) -> dict:
         """Конвертирует в словарь для JSON-сериализации."""
@@ -34,12 +141,26 @@ class GeneratorState:
             "last_batch_id": self.last_batch_id,
             "last_timestamp": self.last_timestamp.isoformat(),
             "version": self.version,
+            "population": self.population,
+            "active_visits": self.active_visits,
+            "pending_visit_births": self.pending_visit_births,
         }
 
     @classmethod
     def from_dict(cls, data: dict) -> "GeneratorState":
         """Создаёт состояние из словаря."""
         try:
+            if not isinstance(data, dict):
+                raise ValueError("state must be an object")
+            version = data.get("version", "1.0")
+            if version != STATE_VERSION:
+                logger.warning(
+                    "Unsupported generator state version %s, will start fresh",
+                    version,
+                )
+                raise ValueError(f"unsupported state version: {version}")
+            _validate_v2_payload(data)
+
             rng_state_raw = data.get("rng_state")
             if not rng_state_raw:
                 logger.warning("State missing rng_state field")
@@ -65,7 +186,10 @@ class GeneratorState:
                 last_timestamp=datetime.fromisoformat(
                     data.get("last_timestamp", "1970-01-01T00:00:00+00:00")
                 ),
-                version=data.get("version", "1.0"),
+                version=version,
+                population=data.get("population", []),
+                active_visits=data.get("active_visits", []),
+                pending_visit_births=data.get("pending_visit_births", 0.0),
             )
         except Exception as e:
             logger.warning(f"Invalid state format, will start fresh: {e}")