feat(generator): инкрементальные счётчики manifest без перечитки Kafka

- Зачем:
  - world_next_day перечитывал всю историю топиков Kafka ради
    накопительных счётчиков — время прогона росло с возрастом мира
    (issue #5, находка F9).
- Что:
  - счётчики засеваются при import из уже прочитанного артефакта и при
    backfill из потока; next-day продвигает их только событиями нового
    дня, полного чтения Kafka больше нет;
  - катящаяся контрольная сумма — сумма SHA-256 событий по модулю 2^256
    (инкремент равен полному пересчёту), старый формат артефакта
    принимается без изменений;
  - точные множества click_id/user_domain_id вынесены из manifest в
    цепочку контент-адресуемых фрагментов (<=10 000 ID, SHA-256-цепочка,
    отдельный топик counter_chunks) — потолок сообщения Kafka не грозит,
    предел 900 000 байт проверяется явно с понятной ошибкой;
  - порядок записи всюду: фрагменты -> manifest -> state; старое локальное
    состояние отклоняется с подсказкой перезапустить import;
  - документация manifest/state обновлена (ARCHITECTURE, OPERATIONS,
    runbook startup-history).
- Проверка:
  - make test (216+31) и make lint зелёные;
  - живая приёмка на чистом стенде: import 235 с; три прогона
    world_next_day — 716/718/716 с (плоское время, O(нового дня));
    мир 3->6 дней, 561 942 события; make generated-history-chain-check —
    все порции и стыки однородны;
  - тест равенства инкремента и полного пересчёта:
    test_incremental_counters_equal_full_recompute.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

generator/src/clickstream_generator/airflow_control.py

@@ -21,8 +21,11 @@ from clickstream_generator.service import GeneratorService
 from clickstream_generator.startup_history_artifact import (
     KafkaRawPublisher,
     KafkaTopicInspector,
+    ManifestCounters,
     compare_clickhouse_stats_to_manifest,
+    cumulative_counter_reference,
     import_startup_history_artifact,
+    known_counter_ids_from_state,
     load_startup_history_artifact,
     manifest_boundaries,
     validate_startup_history_artifact,
@@ -171,6 +174,38 @@ def assert_next_day_snapshot(manifest: dict | None, state) -> None:
         raise RuntimeError(
             f"state не согласован с T_end manifest: {exc}"
         ) from exc
+    counter_reference = state.cumulative_manifest_counters
+    counter_state = manifest.get("cumulative_manifest_counters")
+    if counter_reference is None or counter_state is None:
+        raise RuntimeError(
+            "В state нет накопительных счётчиков manifest; "
+            "повторно запустите import эталонного мира."
+        )
+    if cumulative_counter_reference(counter_state) != counter_reference:
+        raise RuntimeError(
+            "Накопительные счётчики state и manifest не совпадают; "
+            "повторно запустите import эталонного мира."
+        )
+    known_click_ids, known_user_ids = known_counter_ids_from_state(state)
+    try:
+        counters = ManifestCounters.from_state(
+            counter_state,
+            known_click_ids=known_click_ids,
+            known_user_ids=known_user_ids,
+        )
+    except ValueError as exc:
+        raise RuntimeError(
+            "Накопительные счётчики manifest повреждены; "
+            "повторно запустите import эталонного мира."
+        ) from exc
+    if (
+        counters.to_manifest_topics() != manifest.get("topics")
+        or counters.to_manifest_totals() != manifest.get("totals")
+    ):
+        raise RuntimeError(
+            "Накопительные счётчики не совпадают с числами manifest; "
+            "повторно запустите import эталонного мира."
+        )
 
 
 def _parse_utc_timestamp(value: str):

generator/src/clickstream_generator/kafka_io.py

@@ -1,5 +1,6 @@
 """Kafka-интеграция генератора."""
 
+import hashlib
 import json
 import logging
 import sys
@@ -14,6 +15,7 @@ from clickstream_generator.state import UnsupportedStateVersionError
 
 logger = logging.getLogger("generator")
 DATA_TOPICS = ("browser_events", "location_events", "device_events", "geo_events")
+MAX_COMPACT_MESSAGE_BYTES = 900_000
 
 _kafka_imported = False
 KafkaProducer = None
@@ -207,7 +209,9 @@ class KafkaStartupHistoryManifest:
     """Хранение манифеста стартовой истории в Kafka compact topic."""
 
     MANIFEST_TOPIC = "generator_startup_history_manifest"
+    COUNTER_TOPIC = "generator_startup_history_counter_chunks"
     MANIFEST_KEY = "default"
+    COUNTER_CHUNK_KEY_PREFIX = "id-set:"
 
     def __init__(self, bootstrap_servers: str):
         self.bootstrap_servers = bootstrap_servers
@@ -227,6 +231,8 @@ class KafkaStartupHistoryManifest:
 
     def save(self, manifest: dict) -> None:
         """Сохраняет манифест стартовой истории."""
+        self._assert_message_size(manifest, "manifest")
+
         def _do_send():
             self.producer.send(
                 self.MANIFEST_TOPIC,
@@ -236,6 +242,40 @@ class KafkaStartupHistoryManifest:
 
         _retry(_do_send, max_retries=3, base_delay=0.5)
 
+    def save_counter_chunk(self, chunk_sha256: str, chunk: dict) -> None:
+        """Сохраняет неизменяемый фрагмент точных множеств идентификаторов."""
+        if not isinstance(chunk_sha256, str) or len(chunk_sha256) != 64:
+            raise ValueError("хеш фрагмента множеств идентификаторов неверен")
+        actual_sha256 = hashlib.sha256(
+            json.dumps(
+                chunk,
+                sort_keys=True,
+                separators=(",", ":"),
+                ensure_ascii=True,
+            ).encode("utf-8")
+        ).hexdigest()
+        if actual_sha256 != chunk_sha256:
+            raise ValueError("хеш фрагмента множеств идентификаторов не совпадает")
+        self._assert_message_size(chunk, "фрагмент множеств идентификаторов")
+
+        def _do_send():
+            self.producer.send(
+                self.COUNTER_TOPIC,
+                key=self.COUNTER_CHUNK_KEY_PREFIX + chunk_sha256,
+                value=chunk,
+            )
+
+        _retry(_do_send, max_retries=3, base_delay=0.5)
+
+    @staticmethod
+    def _assert_message_size(value: dict, label: str) -> None:
+        size = len(json.dumps(value, ensure_ascii=True).encode("utf-8"))
+        if size > MAX_COMPACT_MESSAGE_BYTES:
+            raise ValueError(
+                f"{label} занимает {size} байт и превышает безопасный предел "
+                f"{MAX_COMPACT_MESSAGE_BYTES} байт одного сообщения Kafka"
+            )
+
     def flush(self) -> None:
         """Сбрасывает буфер с retry."""
         def _do_flush():
@@ -372,8 +412,23 @@ def ensure_topics(bootstrap_servers: str) -> None:
                     "delete.retention.ms": "100",
                 },
             )
+            counter_topic = NewTopic(
+                name=KafkaStartupHistoryManifest.COUNTER_TOPIC,
+                num_partitions=1,
+                replication_factor=1,
+                topic_configs={
+                    "cleanup.policy": "compact",
+                    "min.cleanable.dirty.ratio": "0.1",
+                    "delete.retention.ms": "100",
+                },
+            )
 
-            for topic in [history_topic, state_topic, manifest_topic]:
+            for topic in [
+                history_topic,
+                state_topic,
+                manifest_topic,
+                counter_topic,
+            ]:
                 try:
                     admin_client.create_topics([topic])
                     logger.info(f"Created topic: {topic.name}")

generator/src/clickstream_generator/service.py

@@ -16,7 +16,6 @@ from clickstream_generator.generation import EventGenerator
 from clickstream_generator.kafka_io import (
     BatchRecord,
     KafkaBatchHistory,
-    KafkaDataTopicReader,
     KafkaPublisher,
     KafkaStateManager,
     KafkaStartupHistoryManifest,
@@ -32,7 +31,9 @@ from clickstream_generator.startup_history_artifact import (
     StartupHistoryArtifactBuilder,
     ManifestCounters,
     build_manifest,
+    cumulative_counter_reference,
     generation_settings_from_config,
+    known_counter_ids_from_state,
     manifest_boundaries,
     write_startup_history_artifact,
 )
@@ -58,7 +59,6 @@ class GeneratorService:
         self.history: KafkaBatchHistory | None = None
         self.state_manager: KafkaStateManager | None = None
         self.manifest_manager: KafkaStartupHistoryManifest | None = None
-        self.data_reader: KafkaDataTopicReader | None = None
         self._running = False
         self._stop_requested = False
         self._shutdown_event = threading.Event()
@@ -133,10 +133,7 @@ class GeneratorService:
                     restored_state,
                     model_t_end=model_t_end,
                 )
-                self.data_reader = KafkaDataTopicReader(
-                    self.config.kafka_bootstrap_servers
-                )
-                self._run_next_day(manifest)
+                self._run_next_day(manifest, restored_state)
                 self.stop()
                 return
 
@@ -471,13 +468,24 @@ class GeneratorService:
             model_t0=self.config.model_t0,
             gen_seed=self.config.seed,
         )
+        cumulative_state, counter_chunks = (
+            artifact_builder.counters.to_state_with_chunks()
+        )
+        state.cumulative_manifest_counters = cumulative_counter_reference(
+            cumulative_state
+        )
 
         manifest = build_manifest(
             config=self.config,
             counters=artifact_builder.counters,
             state=state,
+            cumulative_state=cumulative_state,
         )
         if self.manifest_manager:
+            for chunk_sha256, chunk in counter_chunks:
+                self.manifest_manager.save_counter_chunk(chunk_sha256, chunk)
+            if counter_chunks:
+                self.manifest_manager.flush()
             self.manifest_manager.save(manifest)
             self.manifest_manager.flush()
 
@@ -519,7 +527,7 @@ class GeneratorService:
             f"status={status}, sent_counts={sent_counts}"
         )
 
-    def _run_next_day(self, manifest: dict) -> None:
+    def _run_next_day(self, manifest: dict, restored_state) -> None:
         """Доливает ровно 24 модельных часа от границы manifest."""
         if not self.publisher:
             raise RuntimeError("publisher не инициализирован")
@@ -528,6 +536,41 @@ class GeneratorService:
         if not self.state_manager or not self.manifest_manager:
             raise RuntimeError("менеджеры state и manifest не инициализированы")
 
+        counter_reference = restored_state.cumulative_manifest_counters
+        counter_state = manifest.get("cumulative_manifest_counters")
+        if counter_reference is None or counter_state is None:
+            raise IncompatibleStateError(
+                "в state нет накопительных счётчиков manifest; "
+                "повторно запустите import эталонного мира"
+            )
+        if cumulative_counter_reference(counter_state) != counter_reference:
+            raise IncompatibleStateError(
+                "накопительные счётчики state и manifest не совпадают; "
+                "повторно запустите import эталонного мира"
+            )
+        known_click_ids, known_user_ids = known_counter_ids_from_state(
+            restored_state
+        )
+        try:
+            counters = ManifestCounters.from_state(
+                counter_state,
+                known_click_ids=known_click_ids,
+                known_user_ids=known_user_ids,
+            )
+        except ValueError as exc:
+            raise IncompatibleStateError(
+                "накопительные счётчики manifest повреждены; "
+                "повторно запустите import эталонного мира"
+            ) from exc
+        if (
+            counters.to_manifest_topics() != manifest.get("topics")
+            or counters.to_manifest_totals() != manifest.get("totals")
+        ):
+            raise IncompatibleStateError(
+                "накопительные счётчики не совпадают с числами manifest; "
+                "повторно запустите import эталонного мира"
+            )
+
         current_t_end = self._as_aware_utc(
             datetime.fromisoformat(manifest["model_t_end"])
         )
@@ -554,6 +597,7 @@ class GeneratorService:
             )
             total_sent, sent_counts, status = self._publish_batch(batch)
             self._raise_on_next_day_publish_error(batch_id, status, sent_counts)
+            counters.add_batch(batch)
             self._write_batch_history(
                 batch_id=batch_id,
                 started_at=started_at,
@@ -573,6 +617,7 @@ class GeneratorService:
             started_at = datetime.now(timezone.utc)
             total_sent, sent_counts, status = self._publish_batch(final_batch)
             self._raise_on_next_day_publish_error(batch_id, status, sent_counts)
+            counters.add_batch(final_batch)
             self._write_batch_history(
                 batch_id=batch_id,
                 started_at=started_at,
@@ -584,11 +629,6 @@ class GeneratorService:
 
         self.publisher.flush()
         self._model_time = target_t_end
-        reader = self.data_reader or KafkaDataTopicReader(
-            self.config.kafka_bootstrap_servers
-        )
-        counters = ManifestCounters()
-        counters.add_batch(reader.load())
         state_batch_id = self._startup_state_batch_id(counters)
         state = self.stream.to_state(
             tick=self._tick,
@@ -602,6 +642,10 @@ class GeneratorService:
             model_t0=self.config.model_t0,
             gen_seed=self.config.seed,
         )
+        cumulative_state, counter_chunks = counters.to_state_with_chunks()
+        state.cumulative_manifest_counters = cumulative_counter_reference(
+            cumulative_state
+        )
         boundaries = manifest_boundaries(manifest) + [target_t_end.isoformat()]
         updated_manifest = build_manifest(
             self.config,
@@ -609,12 +653,17 @@ class GeneratorService:
             state,
             model_t_end=target_t_end,
             boundaries=boundaries,
+            cumulative_state=cumulative_state,
         )
 
-        self.state_manager.save(state)
-        self.state_manager.flush()
+        for chunk_sha256, chunk in counter_chunks:
+            self.manifest_manager.save_counter_chunk(chunk_sha256, chunk)
+        if counter_chunks:
+            self.manifest_manager.flush()
         self.manifest_manager.save(updated_manifest)
         self.manifest_manager.flush()
+        self.state_manager.save(state)
+        self.state_manager.flush()
 
     def _raise_on_next_day_publish_error(
         self,

generator/src/clickstream_generator/startup_history_artifact.py

@@ -3,10 +3,12 @@
 from __future__ import annotations
 
 import argparse
+import base64
 import hashlib
 import json
 import lzma
 import logging
+import zlib
 from copy import deepcopy
 from datetime import datetime, timezone
 from pathlib import Path
@@ -28,6 +30,11 @@ logger = logging.getLogger("generator")
 ARTIFACT_KIND = "clickstream_generator_startup_history"
 ARTIFACT_VERSION = "1.0"
 TOPICS = ("browser_events", "location_events", "device_events", "geo_events")
+COUNTER_STATE_VERSION = "2.0"
+CHECKSUM_MODULUS = 1 << 256
+ID_SET_CHUNK_VERSION = "1.0"
+ID_SET_CHUNK_SIZE = 10_000
+ID_SET_STORAGE = "manifest-topic-sha256-chain-v1"
 IMPORT_TOPICS = (
     "browser_events",
     "location_events",
@@ -35,6 +42,7 @@ IMPORT_TOPICS = (
     "geo_events",
     KafkaStateManager.STATE_TOPIC,
     KafkaStartupHistoryManifest.MANIFEST_TOPIC,
+    KafkaStartupHistoryManifest.COUNTER_TOPIC,
 )
 
 
@@ -45,17 +53,20 @@ class _TopicManifestStats:
         self.rows = 0
         self.min_event_timestamp: str | None = None
         self.max_event_timestamp: str | None = None
-        self._checksum = hashlib.sha256()
+        self._checksum_sum = 0
 
     @property
     def checksum(self) -> str:
-        return self._checksum.hexdigest()
+        return f"{self._checksum_sum:064x}"
 
     def add(self, event: dict, event_timestamp: str | None = None) -> None:
         self.rows += 1
-        self._checksum.update(
+        event_digest = hashlib.sha256(
             json.dumps(event, sort_keys=True, ensure_ascii=True).encode("utf-8")
-        )
+        ).digest()
+        self._checksum_sum = (
+            self._checksum_sum + int.from_bytes(event_digest, "big")
+        ) % CHECKSUM_MODULUS
         timestamp = (
             event_timestamp
             if event_timestamp is not None
@@ -79,6 +90,55 @@ class _TopicManifestStats:
             "checksum_sha256": self.checksum,
         }
 
+    @classmethod
+    def from_state(cls, payload: dict) -> "_TopicManifestStats":
+        """Восстанавливает накопительные числа одного топика."""
+        if not isinstance(payload, dict):
+            raise ValueError("счётчики топика должны быть объектом")
+        rows = payload.get("rows")
+        checksum = payload.get("checksum_sha256")
+        if isinstance(rows, bool) or not isinstance(rows, int) or rows < 0:
+            raise ValueError("rows в накопительных счётчиках должен быть целым")
+        if not isinstance(checksum, str) or len(checksum) != 64:
+            raise ValueError("checksum_sha256 в накопительных счётчиках неверен")
+        try:
+            checksum_sum = int(checksum, 16)
+        except ValueError as exc:
+            raise ValueError(
+                "checksum_sha256 в накопительных счётчиках неверен"
+            ) from exc
+
+        stats = cls()
+        stats.rows = rows
+        stats.min_event_timestamp = payload.get("min_event_timestamp")
+        stats.max_event_timestamp = payload.get("max_event_timestamp")
+        stats._checksum_sum = checksum_sum
+        return stats
+
+
+class _LegacyTopicManifestStats(_TopicManifestStats):
+    """Прежняя контрольная сумма для проверки неизменённого артефакта."""
+
+    def __init__(self):
+        super().__init__()
+        self._legacy_checksum = hashlib.sha256()
+
+    @property
+    def checksum(self) -> str:
+        return self._legacy_checksum.hexdigest()
+
+    def add(self, event: dict, event_timestamp: str | None = None) -> None:
+        self.rows += 1
+        self._legacy_checksum.update(
+            json.dumps(event, sort_keys=True, ensure_ascii=True).encode("utf-8")
+        )
+        timestamp = (
+            event_timestamp
+            if event_timestamp is not None
+            else event.get("event_timestamp")
+        )
+        self.add_timestamp(timestamp)
+
 
 class ManifestCounters:
     """Счётчики стартовой истории для manifest."""
@@ -87,6 +147,11 @@ class ManifestCounters:
         self.topic_stats = {topic: _TopicManifestStats() for topic in TOPICS}
         self.click_ids: set[str] = set()
         self.user_ids: set[str] = set()
+        self._new_click_ids: set[str] = set()
+        self._new_user_ids: set[str] = set()
+        self._visit_count = 0
+        self._user_count = 0
+        self._previous_id_set_chain = _empty_id_set_chain()
 
     def add_batch(self, batch: dict[str, list[dict]]) -> None:
         browser_events = batch.get("browser_events", [])
@@ -117,11 +182,23 @@ class ManifestCounters:
                         stats.add_timestamp(max(timestamps))
                 stats.add(event, event_timestamp=event_timestamp)
                 click_id = event.get("click_id")
-                if topic == "browser_events" and click_id:
+                if (
+                    topic == "browser_events"
+                    and click_id
+                    and click_id not in self.click_ids
+                ):
                     self.click_ids.add(click_id)
+                    self._new_click_ids.add(click_id)
+                    self._visit_count += 1
                 user_id = event.get("user_domain_id")
-                if topic == "device_events" and user_id:
+                if (
+                    topic == "device_events"
+                    and user_id
+                    and user_id not in self.user_ids
+                ):
                     self.user_ids.add(user_id)
+                    self._new_user_ids.add(user_id)
+                    self._user_count += 1
 
     def to_manifest_topics(self) -> dict:
         return {topic: stats.to_dict() for topic, stats in self.topic_stats.items()}
@@ -130,12 +207,214 @@ class ManifestCounters:
         browser_stats = self.topic_stats["browser_events"]
         return {
             "events": browser_stats.rows,
-            "visits": len(self.click_ids),
-            "users": len(self.user_ids),
+            "visits": self._visit_count,
+            "users": self._user_count,
             "min_event_timestamp": browser_stats.min_event_timestamp,
             "max_event_timestamp": browser_stats.max_event_timestamp,
         }
 
+    def to_state(self) -> dict:
+        """Возвращает JSON-сериализуемое накопительное состояние."""
+        state, _chunks = self.to_state_with_chunks()
+        return state
+
+    def to_state_with_chunks(self) -> tuple[dict, list[tuple[str, dict]]]:
+        """Возвращает компактное состояние и новые фрагменты точных множеств."""
+        id_set_chain, chunks = _extend_id_set_chain(
+            self._previous_id_set_chain,
+            self._new_click_ids,
+            self._new_user_ids,
+        )
+        state = {
+            "version": COUNTER_STATE_VERSION,
+            "checksum_algorithm": "sha256-sum-v1",
+            "topics": self.to_manifest_topics(),
+            "totals": {
+                "visits": self._visit_count,
+                "users": self._user_count,
+            },
+            "id_sets": id_set_chain,
+        }
+        return state, chunks
+
+    @classmethod
+    def from_state(
+        cls,
+        payload: dict,
+        *,
+        known_click_ids: set[str] | None = None,
+        known_user_ids: set[str] | None = None,
+    ) -> "ManifestCounters":
+        """Продолжает счётчики из state без чтения старых событий."""
+        if not isinstance(payload, dict):
+            raise ValueError("накопительные счётчики должны быть объектом")
+        if payload.get("version") != COUNTER_STATE_VERSION:
+            raise ValueError("версия накопительных счётчиков не поддерживается")
+        if payload.get("checksum_algorithm") != "sha256-sum-v1":
+            raise ValueError("алгоритм накопительной контрольной суммы не поддерживается")
+        topic_payloads = payload.get("topics")
+        if not isinstance(topic_payloads, dict) or set(topic_payloads) != set(TOPICS):
+            raise ValueError("в накопительных счётчиках нет всех топиков")
+        totals = payload.get("totals")
+        if not isinstance(totals, dict):
+            raise ValueError("в накопительных счётчиках нет итогов")
+        visits = _non_negative_int(totals.get("visits"), "visits")
+        users = _non_negative_int(totals.get("users"), "users")
+        id_sets = payload.get("id_sets")
+        _validate_id_set_chain(id_sets, visits=visits, users=users)
+
+        counters = cls()
+        counters.topic_stats = {
+            topic: _TopicManifestStats.from_state(topic_payloads[topic])
+            for topic in TOPICS
+        }
+        counters.click_ids = set(known_click_ids or ())
+        counters.user_ids = set(known_user_ids or ())
+        counters._new_click_ids = set()
+        counters._new_user_ids = set()
+        counters._visit_count = visits
+        counters._user_count = users
+        counters._previous_id_set_chain = dict(id_sets)
+        return counters
+
+
+def cumulative_counter_reference(counter_state: dict) -> dict:
+    """Строит компактную ссылку state на полный набор из manifest."""
+    canonical = json.dumps(
+        counter_state,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=True,
+    ).encode("utf-8")
+    return {
+        "version": COUNTER_STATE_VERSION,
+        "manifest_sha256": hashlib.sha256(canonical).hexdigest(),
+    }
+
+
+def known_counter_ids_from_state(state: GeneratorState) -> tuple[set[str], set[str]]:
+    """Возвращает идентификаторы, которые уже успели попасть в data-топики."""
+    click_ids = {visit["click_id"] for visit in state.active_visits}
+    user_ids = {
+        user["user_domain_id"]
+        for user in state.population
+        if user.get("active_click_id") is not None
+        or user.get("last_finished_at") is not None
+    }
+    return click_ids, user_ids
+
+
+def _empty_id_set_chain() -> dict:
+    return {
+        "storage": ID_SET_STORAGE,
+        "latest_sha256": None,
+        "chunks": 0,
+        "click_ids": 0,
+        "user_ids": 0,
+    }
+
+
+def _non_negative_int(value: Any, label: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+        raise ValueError(f"{label} в накопительных счётчиках должен быть целым")
+    return value
+
+
+def _validate_id_set_chain(payload: Any, *, visits: int, users: int) -> None:
+    if not isinstance(payload, dict):
+        raise ValueError("в накопительных счётчиках нет множеств идентификаторов")
+    if payload.get("storage") != ID_SET_STORAGE:
+        raise ValueError("хранилище множеств идентификаторов не поддерживается")
+    chunks = _non_negative_int(payload.get("chunks"), "chunks")
+    click_ids = _non_negative_int(payload.get("click_ids"), "click_ids")
+    user_ids = _non_negative_int(payload.get("user_ids"), "user_ids")
+    latest = payload.get("latest_sha256")
+    if latest is not None:
+        if not isinstance(latest, str) or len(latest) != 64:
+            raise ValueError("ссылка на фрагмент множеств идентификаторов неверна")
+        try:
+            int(latest, 16)
+        except ValueError as exc:
+            raise ValueError(
+                "ссылка на фрагмент множеств идентификаторов неверна"
+            ) from exc
+    if (chunks == 0) != (latest is None):
+        raise ValueError("цепочка множеств идентификаторов повреждена")
+    if click_ids != visits or user_ids != users:
+        raise ValueError("размеры множеств идентификаторов не совпадают с итогами")
+
+
+def _extend_id_set_chain(
+    previous: dict,
+    click_ids: set[str],
+    user_ids: set[str],
+) -> tuple[dict, list[tuple[str, dict]]]:
+    _validate_id_set_chain(
+        previous,
+        visits=_non_negative_int(previous.get("click_ids"), "click_ids"),
+        users=_non_negative_int(previous.get("user_ids"), "user_ids"),
+    )
+    tagged_ids = [("click_id", value) for value in sorted(click_ids)]
+    tagged_ids.extend(("user_id", value) for value in sorted(user_ids))
+    previous_sha256 = previous["latest_sha256"]
+    chunk_count = previous["chunks"]
+    chunks: list[tuple[str, dict]] = []
+    for offset in range(0, len(tagged_ids), ID_SET_CHUNK_SIZE):
+        part = tagged_ids[offset : offset + ID_SET_CHUNK_SIZE]
+        part_click_ids = {
+            value for kind, value in part if kind == "click_id"
+        }
+        part_user_ids = {
+            value for kind, value in part if kind == "user_id"
+        }
+        chunk = {
+            "version": ID_SET_CHUNK_VERSION,
+            "encoding": "json-zlib-base64-v1",
+            "sequence": chunk_count + 1,
+            "previous_sha256": previous_sha256,
+            "click_ids": _encode_id_set(part_click_ids),
+            "user_ids": _encode_id_set(part_user_ids),
+        }
+        digest = hashlib.sha256(_canonical_json(chunk)).hexdigest()
+        chunks.append((digest, chunk))
+        previous_sha256 = digest
+        chunk_count += 1
+
+    return (
+        {
+            "storage": ID_SET_STORAGE,
+            "latest_sha256": previous_sha256,
+            "chunks": chunk_count,
+            "click_ids": previous["click_ids"] + len(click_ids),
+            "user_ids": previous["user_ids"] + len(user_ids),
+        },
+        chunks,
+    )
+
+
+def _canonical_json(value: dict) -> bytes:
+    return json.dumps(
+        value,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=True,
+    ).encode("utf-8")
+
+
+def _encode_id_set(values: set[str]) -> str:
+    raw = json.dumps(sorted(values), separators=(",", ":")).encode("utf-8")
+    return base64.b64encode(zlib.compress(raw, level=9)).decode("ascii")
+
+
+class _LegacyManifestCounters(ManifestCounters):
+    """Полный пересчёт старой суммы только при проверке артефакта."""
+
+    def __init__(self):
+        super().__init__()
+        self.topic_stats = {
+            topic: _LegacyTopicManifestStats() for topic in TOPICS
+        }
+
 
 class StartupHistoryArtifactBuilder:
     """Собирает сообщения топиков для портативного файла."""
@@ -158,12 +437,16 @@ class StartupHistoryArtifactBuilder:
                 )
 
     def to_artifact(self, manifest: dict, state: GeneratorState) -> dict:
+        artifact_manifest = deepcopy(manifest)
+        artifact_manifest.pop("cumulative_manifest_counters", None)
+        artifact_state = state.to_dict()
+        artifact_state.pop("cumulative_manifest_counters", None)
         return {
             "artifact_version": ARTIFACT_VERSION,
             "kind": ARTIFACT_KIND,
             "created_at": datetime.now(timezone.utc).isoformat(),
-            "manifest": deepcopy(manifest),
-            "state": state.to_dict(),
+            "manifest": artifact_manifest,
+            "state": artifact_state,
             "topics": deepcopy(self.topics),
             "raw_topics": deepcopy(self.raw_topics),
         }
@@ -295,6 +578,7 @@ def build_manifest(
     *,
     model_t_end: datetime | None = None,
     boundaries: list[str] | None = None,
+    cumulative_state: dict | None = None,
 ) -> dict:
     """Строит manifest стартовой истории."""
     selected_t_end = model_t_end or config.model_t_end
@@ -325,6 +609,7 @@ def build_manifest(
         },
         "topics": counters.to_manifest_topics(),
         "totals": counters.to_manifest_totals(),
+        "cumulative_manifest_counters": cumulative_state or counters.to_state(),
     }
     manifest_boundaries(manifest)
     return manifest
@@ -437,6 +722,16 @@ def import_startup_history_artifact(
         artifact,
         expected_config=expected_config,
     )
+    counters = ManifestCounters()
+    counters.add_batch(topics)
+    cumulative_state, counter_chunks = counters.to_state_with_chunks()
+    state.cumulative_manifest_counters = cumulative_counter_reference(
+        cumulative_state
+    )
+    manifest = deepcopy(manifest)
+    manifest["topics"] = counters.to_manifest_topics()
+    manifest["totals"] = counters.to_manifest_totals()
+    manifest["cumulative_manifest_counters"] = cumulative_state
     if topic_inspector is not None:
         topic_inspector.assert_data_topics_empty()
         snapshot = (
@@ -471,6 +766,10 @@ def import_startup_history_artifact(
             sent_by_topic[topic] = sent
 
         publisher.flush()
+        for chunk_sha256, chunk in counter_chunks:
+            manifest_manager.save_counter_chunk(chunk_sha256, chunk)
+        if counter_chunks:
+            manifest_manager.flush()
         manifest_manager.save(manifest)
         manifest_manager.flush()
         state_manager.save(state)
@@ -548,10 +847,15 @@ def _validate_manifest_topics(manifest: dict, topics: dict[str, list[dict]]) ->
     counters.add_batch({topic: topics[topic] for topic in TOPICS})
     expected_topics = manifest.get("topics") or {}
     expected_totals = manifest.get("totals") or {}
-    if counters.to_manifest_topics() != expected_topics:
-        raise ValueError("startup history artifact mismatch: topic checksums")
     if counters.to_manifest_totals() != expected_totals:
         raise ValueError("startup history artifact mismatch: totals")
+    if counters.to_manifest_topics() == expected_topics:
+        return
+
+    legacy_counters = _LegacyManifestCounters()
+    legacy_counters.add_batch({topic: topics[topic] for topic in TOPICS})
+    if legacy_counters.to_manifest_topics() != expected_topics:
+        raise ValueError("startup history artifact mismatch: topic checksums")
 
 
 def _validate_raw_topics(

generator/src/clickstream_generator/state.py

@@ -204,6 +204,7 @@ class GeneratorState:
     population: list[dict] = field(default_factory=list)
     active_visits: list[dict] = field(default_factory=list)
     pending_visit_births: float = 0.0
+    cumulative_manifest_counters: dict | None = None
 
     def __post_init__(self) -> None:
         if self.model_timestamp is None:
@@ -221,7 +222,7 @@ class GeneratorState:
 
     def to_dict(self) -> dict:
         """Конвертирует в словарь для JSON-сериализации."""
-        return {
+        payload = {
             "tick": self.tick,
             "rng_state": self.rng_state,
             "last_batch_id": self.last_batch_id,
@@ -237,6 +238,11 @@ class GeneratorState:
             "active_visits": self.active_visits,
             "pending_visit_births": self.pending_visit_births,
         }
+        if self.cumulative_manifest_counters is not None:
+            payload["cumulative_manifest_counters"] = (
+                self.cumulative_manifest_counters
+            )
+        return payload
 
     @classmethod
     def from_dict(cls, data: dict) -> "GeneratorState":
@@ -290,6 +296,9 @@ class GeneratorState:
                 population=data.get("population", []),
                 active_visits=data.get("active_visits", []),
                 pending_visit_births=data.get("pending_visit_births", 0.0),
+                cumulative_manifest_counters=data.get(
+                    "cumulative_manifest_counters"
+                ),
             )
         except UnsupportedStateVersionError:
             raise