feat(generator): инкрементальные счётчики manifest без перечитки Kafka

- Зачем:
  - world_next_day перечитывал всю историю топиков Kafka ради
    накопительных счётчиков — время прогона росло с возрастом мира
    (issue #5, находка F9).
- Что:
  - счётчики засеваются при import из уже прочитанного артефакта и при
    backfill из потока; next-day продвигает их только событиями нового
    дня, полного чтения Kafka больше нет;
  - катящаяся контрольная сумма — сумма SHA-256 событий по модулю 2^256
    (инкремент равен полному пересчёту), старый формат артефакта
    принимается без изменений;
  - точные множества click_id/user_domain_id вынесены из manifest в
    цепочку контент-адресуемых фрагментов (<=10 000 ID, SHA-256-цепочка,
    отдельный топик counter_chunks) — потолок сообщения Kafka не грозит,
    предел 900 000 байт проверяется явно с понятной ошибкой;
  - порядок записи всюду: фрагменты -> manifest -> state; старое локальное
    состояние отклоняется с подсказкой перезапустить import;
  - документация manifest/state обновлена (ARCHITECTURE, OPERATIONS,
    runbook startup-history).
- Проверка:
  - make test (216+31) и make lint зелёные;
  - живая приёмка на чистом стенде: import 235 с; три прогона
    world_next_day — 716/718/716 с (плоское время, O(нового дня));
    мир 3->6 дней, 561 942 события; make generated-history-chain-check —
    все порции и стыки однородны;
  - тест равенства инкремента и полного пересчёта:
    test_incremental_counters_equal_full_recompute.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

generator/tests/test_service.py

@@ -25,6 +25,7 @@ from generator import (
     main,
 )
 from clickstream_generator.state import UnsupportedStateVersionError
+from clickstream_generator.service import IncompatibleStateError
 
 
 class TestBatchRecordWithDictConversion:
@@ -425,6 +426,10 @@ class TestGeneratorServiceBackfill:
         self, base_config
     ):
         """Backfill пишет [T0, T_end), state на T_end и повторяемый manifest."""
+        from clickstream_generator.startup_history_artifact import (
+            cumulative_counter_reference,
+        )
+
         model_t0 = datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc)
         model_t_end = model_t0 + timedelta(minutes=3)
         config = replace(
@@ -463,6 +468,11 @@ class TestGeneratorServiceBackfill:
         assert manifest["model_t0"] == model_t0.isoformat()
         assert manifest["model_t_end"] == model_t_end.isoformat()
         assert manifest["state"]["last_batch_id"] == saved_state.last_batch_id
+        assert saved_state.cumulative_manifest_counters == (
+            cumulative_counter_reference(
+                manifest["cumulative_manifest_counters"]
+            )
+        )
         assert manifest["topics"]["browser_events"]["rows"] == len(browser_events)
         for topic in (
             "browser_events",
@@ -516,6 +526,8 @@ class TestGeneratorServiceBackfill:
         self._run_backfill(config)
 
         artifact = load_startup_history_artifact(artifact_path)
+        assert "cumulative_manifest_counters" not in artifact["state"]
+        assert "cumulative_manifest_counters" not in artifact["manifest"]
         state, manifest, topics, _ = validate_startup_history_artifact(
             artifact,
             expected_config=config,
@@ -960,7 +972,8 @@ class TestGeneratorServiceBackfill:
         with pytest.raises(RuntimeError, match="flush down"):
             service._run_backfill()
 
-        service.manifest_manager.save.assert_called_once()
+        service.manifest_manager.save_counter_chunk.assert_called()
+        service.manifest_manager.save.assert_not_called()
         service.manifest_manager.flush.assert_called_once()
         service.state_manager.save.assert_not_called()
         service.state_manager.flush.assert_not_called()
@@ -1008,13 +1021,14 @@ class TestGeneratorServiceBackfill:
 class TestGeneratorServiceNextDay:
     """Проверки ограниченной доливки следующего модельного дня."""
 
-    def test_next_day_publishes_24h_then_state_then_cumulative_manifest(
+    def test_next_day_publishes_24h_then_chunks_manifest_and_state(
         self, base_config
     ):
-        """Next-day пишет [T_end, T_end+24h), затем state и manifest."""
+        """Next-day пишет день, фрагменты множеств, manifest и затем state."""
         from clickstream_generator.startup_history_artifact import (
             StartupHistoryArtifactBuilder,
             build_manifest,
+            cumulative_counter_reference,
         )
 
         model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
@@ -1062,6 +1076,9 @@ class TestGeneratorServiceNextDay:
         }
         initial_builder = StartupHistoryArtifactBuilder()
         initial_builder.add_batch(old_batch)
+        state.cumulative_manifest_counters = cumulative_counter_reference(
+            initial_builder.counters.to_state()
+        )
         manifest = build_manifest(config, initial_builder.counters, state)
 
         published = {topic: [] for topic in old_batch}
@@ -1078,21 +1095,21 @@ class TestGeneratorServiceNextDay:
         service.state_manager = MagicMock()
         service.state_manager.save.side_effect = lambda _state: order.append("state")
         service.manifest_manager = MagicMock()
+        service.manifest_manager.save_counter_chunk.side_effect = (
+            lambda _chunk_sha256, _chunk: order.append("counter_chunk")
+        )
         service.manifest_manager.save.side_effect = (
             lambda _manifest: order.append("manifest")
         )
 
         class DataReader:
             def load(self):
-                return {
-                    topic: old_batch[topic] + published[topic]
-                    for topic in old_batch
-                }
+                raise AssertionError("next-day не должен перечитывать Kafka")
 
         service.data_reader = DataReader()
         service.restore_from_startup_history(state, model_t_end=current_t_end)
 
-        service._run_next_day(manifest)
+        service._run_next_day(manifest, state)
 
         browser_events = published["browser_events"]
         timestamps = [
@@ -1114,18 +1131,21 @@ class TestGeneratorServiceNextDay:
             target_t_end.isoformat(),
         ]
         assert saved_manifest["totals"]["events"] == len(browser_events) + 1
-        assert order == ["data", "state", "manifest"]
+        assert order == ["data", "counter_chunk", "manifest", "state"]
 
         first_day_events = len(browser_events)
         first_day_checksum = saved_manifest["topics"]["browser_events"][
             "checksum_sha256"
         ]
-        service._run_next_day(saved_manifest)
+        service._run_next_day(saved_manifest, saved_state)
 
         second_state = service.state_manager.save.call_args.args[0]
         second_manifest = service.manifest_manager.save.call_args.args[0]
         expected_builder = StartupHistoryArtifactBuilder()
-        expected_builder.add_batch(DataReader().load())
+        expected_builder.add_batch({
+            topic: old_batch[topic] + published[topic]
+            for topic in old_batch
+        })
         assert second_state.model_timestamp == target_t_end + timedelta(hours=24)
         assert second_manifest["boundaries"] == [
             model_t0.isoformat(),
@@ -1140,15 +1160,57 @@ class TestGeneratorServiceNextDay:
             != first_day_checksum
         )
         assert second_manifest["topics"] == expected_builder.counters.to_manifest_topics()
+        assert second_manifest["totals"] == expected_builder.counters.to_manifest_totals()
+        assert second_state.cumulative_manifest_counters == (
+            cumulative_counter_reference(
+                second_manifest["cumulative_manifest_counters"]
+            )
+        )
         assert order == [
             "data",
-            "state",
+            "counter_chunk",
             "manifest",
+            "state",
             "data",
-            "state",
+            "counter_chunk",
             "manifest",
+            "state",
         ]
 
+    def test_next_day_requires_cumulative_counters_from_import(self, base_config):
+        """Старый локальный state требует повторного импорта, а не чтения Kafka."""
+        model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
+        config = replace(
+            base_config,
+            run_mode="next-day",
+            model_t0=model_t0,
+            model_t_end=model_t0 + timedelta(hours=1),
+        )
+        service = GeneratorService(config)
+        state = service.stream.to_state(
+            tick=1,
+            rng_state=service.generator.rng.getstate(),
+            last_batch_id="startup-history-old",
+            last_timestamp=config.model_t_end,
+            model_timestamp=config.model_t_end,
+            model_t0=config.model_t0,
+        )
+        service.publisher = MagicMock()
+        service.history = MagicMock()
+        service.state_manager = MagicMock()
+        service.manifest_manager = MagicMock()
+        service._model_time = config.model_t_end
+        manifest = {
+            "model_t0": model_t0.isoformat(),
+            "model_t_end": config.model_t_end.isoformat(),
+            "boundaries": [model_t0.isoformat(), config.model_t_end.isoformat()],
+        }
+
+        with pytest.raises(IncompatibleStateError, match="повторно запустите import"):
+            service._run_next_day(manifest, state)
+
+        service.publisher.publish.assert_not_called()
+
     def test_next_day_publish_error_does_not_move_state_or_manifest(self, base_config):
         """Ошибка data-топика оставляет обе точки фиксации без изменений."""
         model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
@@ -1177,8 +1239,29 @@ class TestGeneratorServiceNextDay:
             "boundaries": [model_t0.isoformat(), config.model_t_end.isoformat()],
         }
 
+        from clickstream_generator.startup_history_artifact import (
+            ManifestCounters,
+            cumulative_counter_reference,
+        )
+
+        state = service.stream.to_state(
+            tick=1,
+            rng_state=service.generator.rng.getstate(),
+            last_batch_id="startup-history-test",
+            last_timestamp=config.model_t_end,
+            model_timestamp=config.model_t_end,
+            model_t0=config.model_t0,
+        )
+        empty_counters = ManifestCounters()
+        manifest["cumulative_manifest_counters"] = empty_counters.to_state()
+        manifest["topics"] = empty_counters.to_manifest_topics()
+        manifest["totals"] = empty_counters.to_manifest_totals()
+        state.cumulative_manifest_counters = cumulative_counter_reference(
+            manifest["cumulative_manifest_counters"]
+        )
+
         with pytest.raises(RuntimeError, match="Публикация next-day не удалась"):
-            service._run_next_day(manifest)
+            service._run_next_day(manifest, state)
 
         service.state_manager.save.assert_not_called()
         service.manifest_manager.save.assert_not_called()