feat(generator): инкрементальные счётчики manifest без перечитки Kafka

- Зачем:
  - world_next_day перечитывал всю историю топиков Kafka ради
    накопительных счётчиков — время прогона росло с возрастом мира
    (issue #5, находка F9).
- Что:
  - счётчики засеваются при import из уже прочитанного артефакта и при
    backfill из потока; next-day продвигает их только событиями нового
    дня, полного чтения Kafka больше нет;
  - катящаяся контрольная сумма — сумма SHA-256 событий по модулю 2^256
    (инкремент равен полному пересчёту), старый формат артефакта
    принимается без изменений;
  - точные множества click_id/user_domain_id вынесены из manifest в
    цепочку контент-адресуемых фрагментов (<=10 000 ID, SHA-256-цепочка,
    отдельный топик counter_chunks) — потолок сообщения Kafka не грозит,
    предел 900 000 байт проверяется явно с понятной ошибкой;
  - порядок записи всюду: фрагменты -> manifest -> state; старое локальное
    состояние отклоняется с подсказкой перезапустить import;
  - документация manifest/state обновлена (ARCHITECTURE, OPERATIONS,
    runbook startup-history).
- Проверка:
  - make test (216+31) и make lint зелёные;
  - живая приёмка на чистом стенде: import 235 с; три прогона
    world_next_day — 716/718/716 с (плоское время, O(нового дня));
    мир 3->6 дней, 561 942 события; make generated-history-chain-check —
    все порции и стыки однородны;
  - тест равенства инкремента и полного пересчёта:
    test_incremental_counters_equal_full_recompute.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

generator/tests/test_startup_history_artifact.py

@@ -4,6 +4,7 @@
 
 from dataclasses import replace
 from datetime import datetime, timezone
+import hashlib
 import json
 import lzma
 
@@ -110,6 +111,144 @@ def _batch() -> dict[str, list[dict]]:
     }
 
 
+def test_incremental_counters_equal_full_recompute():
+    """Накопление по частям даёт те же числа, что полный пересчёт мира."""
+    from clickstream_generator.startup_history_artifact import ManifestCounters
+
+    first_day = _batch()
+    second_day = {
+        "browser_events": [
+            {
+                "event_id": "event-2",
+                "click_id": "click-1",
+                "user_domain_id": "user-1",
+                "event_timestamp": "2026-01-02 00:00:00.000000",
+            },
+            {
+                "event_id": "event-3",
+                "click_id": "click-2",
+                "user_domain_id": "user-2",
+                "event_timestamp": "2026-01-02 00:01:00.000000",
+            },
+        ],
+        "location_events": [
+            {"event_id": "event-2", "page_url_path": "/cart"},
+            {"event_id": "event-3", "page_url_path": "/home"},
+        ],
+        "device_events": [
+            {"click_id": "click-1", "user_domain_id": "user-1"},
+            {"click_id": "click-2", "user_domain_id": "user-2"},
+        ],
+        "geo_events": [
+            {"click_id": "click-1", "country": "RU"},
+            {"click_id": "click-2", "country": "KZ"},
+        ],
+    }
+
+    incremental = ManifestCounters()
+    incremental.add_batch(first_day)
+    incremental = ManifestCounters.from_state(
+        incremental.to_state(),
+        known_click_ids={"click-1"},
+        known_user_ids={"user-1"},
+    )
+    incremental.add_batch(second_day)
+
+    full = ManifestCounters()
+    full.add_batch({
+        topic: first_day[topic] + second_day[topic]
+        for topic in first_day
+    })
+
+    assert incremental.to_manifest_topics() == full.to_manifest_topics()
+    assert incremental.to_manifest_totals() == full.to_manifest_totals()
+
+
+def test_exact_id_sets_are_split_into_bounded_hash_chain(monkeypatch):
+    """Manifest ссылается на цепочку малых точных фрагментов, а не растёт сам."""
+    from clickstream_generator import startup_history_artifact as module
+
+    monkeypatch.setattr(module, "ID_SET_CHUNK_SIZE", 2)
+    counters = module.ManifestCounters()
+    batch = _batch()
+    batch["browser_events"].extend([
+        {
+            "event_id": "event-2",
+            "click_id": "click-2",
+            "event_timestamp": "2026-01-01 00:01:00.000000",
+        },
+        {
+            "event_id": "event-3",
+            "click_id": "click-3",
+            "event_timestamp": "2026-01-01 00:02:00.000000",
+        },
+    ])
+    counters.add_batch(batch)
+
+    state, chunks = counters.to_state_with_chunks()
+
+    assert len(chunks) == 2
+    assert state["totals"] == {"visits": 3, "users": 1}
+    assert state["id_sets"] == {
+        "storage": "manifest-topic-sha256-chain-v1",
+        "latest_sha256": chunks[-1][0],
+        "chunks": 2,
+        "click_ids": 3,
+        "user_ids": 1,
+    }
+    assert chunks[0][1]["previous_sha256"] is None
+    assert chunks[1][1]["previous_sha256"] == chunks[0][0]
+    assert "click-1" not in json.dumps(state)
+
+    restored = module.ManifestCounters.from_state(
+        state,
+        known_click_ids={"click-3"},
+        known_user_ids={"user-1"},
+    )
+    restored.add_batch({
+        "browser_events": [
+            {
+                "event_id": "event-4",
+                "click_id": "click-3",
+                "event_timestamp": "2026-01-02 00:00:00.000000",
+            },
+            {
+                "event_id": "event-5",
+                "click_id": "click-4",
+                "event_timestamp": "2026-01-02 00:01:00.000000",
+            },
+        ],
+        "location_events": [],
+        "device_events": [{"click_id": "click-4", "user_domain_id": "user-2"}],
+        "geo_events": [],
+    })
+    next_state, next_chunks = restored.to_state_with_chunks()
+
+    assert next_state["totals"] == {"visits": 4, "users": 2}
+    assert len(next_chunks) == 1
+    assert next_chunks[0][1]["previous_sha256"] == chunks[-1][0]
+    assert next_state["id_sets"]["chunks"] == 3
+
+
+def test_counter_chunk_has_explicit_kafka_size_and_hash_guards():
+    """Большой или неверно адресованный фрагмент падает до отправки в Kafka."""
+    from clickstream_generator.kafka_io import (
+        MAX_COMPACT_MESSAGE_BYTES,
+        KafkaStartupHistoryManifest,
+    )
+
+    manager = object.__new__(KafkaStartupHistoryManifest)
+    with pytest.raises(ValueError, match="безопасный предел"):
+        manager._assert_message_size(
+            {"payload": "x" * MAX_COMPACT_MESSAGE_BYTES},
+            "фрагмент",
+        )
+    with pytest.raises(ValueError, match="хеш.*не совпадает"):
+        manager.save_counter_chunk("0" * 64, {"version": "1.0"})
+
+    assert manager.COUNTER_TOPIC != manager.MANIFEST_TOPIC
+
+
 def test_artifact_roundtrip_keeps_events_state_and_manifest(base_config):
     """Артефакт хранит события, state и manifest одним проверяемым набором."""
     from clickstream_generator.startup_history_artifact import (
@@ -173,6 +312,43 @@ def test_artifact_roundtrip_keeps_events_state_and_manifest(base_config):
         path.unlink(missing_ok=True)
 
 
+def test_legacy_artifact_checksum_remains_valid(base_config):
+    """Эталонный артефакт со старой суммой не требует пересборки."""
+    from clickstream_generator.startup_history_artifact import (
+        StartupHistoryArtifactBuilder,
+        build_manifest,
+        validate_startup_history_artifact,
+    )
+
+    state = _state()
+    config = replace(
+        base_config,
+        model_t0=state.model_t0,
+        model_t_end=state.model_timestamp,
+        model_time_speed=1,
+        model_timezone="UTC",
+        seed=42,
+    )
+    builder = StartupHistoryArtifactBuilder()
+    builder.add_batch(_batch())
+    manifest = build_manifest(config, builder.counters, state)
+    for topic, events in builder.topics.items():
+        checksum = hashlib.sha256()
+        for event in events:
+            checksum.update(
+                json.dumps(event, sort_keys=True, ensure_ascii=True).encode("utf-8")
+            )
+        manifest["topics"][topic]["checksum_sha256"] = checksum.hexdigest()
+
+    artifact = builder.to_artifact(manifest=manifest, state=state)
+
+    _, loaded_manifest, _, _ = validate_startup_history_artifact(
+        artifact,
+        expected_config=config,
+    )
+    assert loaded_manifest["topics"] == artifact["manifest"]["topics"]
+
+
 def test_manifest_and_artifact_show_launch_profile(base_config):
     """Manifest и файл артефакта показывают выбранный профиль запуска."""
     from clickstream_generator.startup_history_artifact import (
@@ -372,6 +548,8 @@ def test_raw_topic_value_can_keep_original_json_bytes(base_config):
         manifest=build_manifest(config=config, counters=builder.counters, state=state),
         state=state,
     )
+    assert "cumulative_manifest_counters" not in artifact["state"]
+    assert "cumulative_manifest_counters" not in artifact["manifest"]
     raw_value = (
         '{"event_timestamp":"2026-01-01 00:00:00.000000",'
         '"user_domain_id":"user-1","click_id":"click-1","event_id":"event-1"}'
@@ -391,9 +569,15 @@ def test_raw_topic_value_can_keep_original_json_bytes(base_config):
             return None
 
     class CompactWriter:
+        def __init__(self):
+            self.chunks = []
+
         def save(self, value):
             self.saved = value
 
+        def save_counter_chunk(self, chunk_sha256, chunk):
+            self.chunks.append((chunk_sha256, chunk))
+
         def flush(self):
             return None
 
@@ -609,8 +793,10 @@ def test_import_rolls_back_kafka_topics_after_partial_publish(base_config):
 def test_import_replays_events_and_compact_topics(base_config):
     """Импорт пишет события в Kafka и служебные compact-топики, не трогая ClickHouse."""
     from clickstream_generator.startup_history_artifact import (
+        ManifestCounters,
         StartupHistoryArtifactBuilder,
         build_manifest,
+        cumulative_counter_reference,
         import_startup_history_artifact,
     )
 
@@ -646,10 +832,14 @@ def test_import_replays_events_and_compact_topics(base_config):
         def __init__(self):
             self.saved = None
             self.flushed = False
+            self.chunks = []
 
         def save(self, value):
             self.saved = value
 
+        def save_counter_chunk(self, chunk_sha256, chunk):
+            self.chunks.append((chunk_sha256, chunk))
+
         def flush(self):
             self.flushed = True
 
@@ -678,7 +868,27 @@ def test_import_replays_events_and_compact_topics(base_config):
     )
     assert result["events"] == 4
     assert state_manager.saved.last_batch_id == state.last_batch_id
+    assert state_manager.saved.cumulative_manifest_counters is not None
     assert manifest_manager.saved["state"]["last_batch_id"] == state.last_batch_id
+    assert state_manager.saved.cumulative_manifest_counters == (
+        cumulative_counter_reference(
+            manifest_manager.saved["cumulative_manifest_counters"]
+        )
+    )
+    seeded = ManifestCounters.from_state(
+        manifest_manager.saved["cumulative_manifest_counters"],
+        known_click_ids={"click-1"},
+        known_user_ids={"user-1"},
+    )
+    assert seeded.click_ids == {"click-1"}
+    assert seeded.user_ids == {"user-1"}
+    assert len(manifest_manager.chunks) == 1
+    assert (
+        manifest_manager.saved["cumulative_manifest_counters"]["id_sets"][
+            "latest_sha256"
+        ]
+        == manifest_manager.chunks[0][0]
+    )
     assert publisher.flushed
     assert state_manager.flushed
     assert manifest_manager.flushed