"""Портативный артефакт стартовой истории."""

from __future__ import annotations

import argparse
import base64
import hashlib
import json
import lzma
import logging
import zlib
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from clickstream_generator.config import Config
from clickstream_generator.kafka_io import (
    KafkaStateManager,
    KafkaStartupHistoryManifest,
    ensure_topics,
    kafka_event_value_json,
    _kafka_importer,
)
from clickstream_generator.state import GeneratorState


logger = logging.getLogger("generator")

ARTIFACT_KIND = "clickstream_generator_startup_history"
ARTIFACT_VERSION = "1.0"
TOPICS = ("browser_events", "location_events", "device_events", "geo_events")
COUNTER_STATE_VERSION = "2.0"
CHECKSUM_MODULUS = 1 << 256
ID_SET_CHUNK_VERSION = "1.0"
ID_SET_CHUNK_SIZE = 10_000
ID_SET_STORAGE = "manifest-topic-sha256-chain-v1"
IMPORT_TOPICS = (
    "browser_events",
    "location_events",
    "device_events",
    "geo_events",
    KafkaStateManager.STATE_TOPIC,
    KafkaStartupHistoryManifest.MANIFEST_TOPIC,
    KafkaStartupHistoryManifest.COUNTER_TOPIC,
)


class _TopicManifestStats:
    """Накопительные счётчики одного Kafka-топика для manifest."""

    def __init__(self):
        self.rows = 0
        self.min_event_timestamp: str | None = None
        self.max_event_timestamp: str | None = None
        self._checksum_sum = 0

    @property
    def checksum(self) -> str:
        return f"{self._checksum_sum:064x}"

    def add(self, event: dict, event_timestamp: str | None = None) -> None:
        self.rows += 1
        event_digest = hashlib.sha256(
            json.dumps(event, sort_keys=True, ensure_ascii=True).encode("utf-8")
        ).digest()
        self._checksum_sum = (
            self._checksum_sum + int.from_bytes(event_digest, "big")
        ) % CHECKSUM_MODULUS
        timestamp = (
            event_timestamp
            if event_timestamp is not None
            else event.get("event_timestamp")
        )
        self.add_timestamp(timestamp)

    def add_timestamp(self, timestamp: str | None) -> None:
        if timestamp is None:
            return
        if self.min_event_timestamp is None or timestamp < self.min_event_timestamp:
            self.min_event_timestamp = timestamp
        if self.max_event_timestamp is None or timestamp > self.max_event_timestamp:
            self.max_event_timestamp = timestamp

    def to_dict(self) -> dict:
        return {
            "rows": self.rows,
            "min_event_timestamp": self.min_event_timestamp,
            "max_event_timestamp": self.max_event_timestamp,
            "checksum_sha256": self.checksum,
        }

    @classmethod
    def from_state(cls, payload: dict) -> "_TopicManifestStats":
        """Восстанавливает накопительные числа одного топика."""
        if not isinstance(payload, dict):
            raise ValueError("счётчики топика должны быть объектом")
        rows = payload.get("rows")
        checksum = payload.get("checksum_sha256")
        if isinstance(rows, bool) or not isinstance(rows, int) or rows < 0:
            raise ValueError("rows в накопительных счётчиках должен быть целым")
        if not isinstance(checksum, str) or len(checksum) != 64:
            raise ValueError("checksum_sha256 в накопительных счётчиках неверен")
        try:
            checksum_sum = int(checksum, 16)
        except ValueError as exc:
            raise ValueError(
                "checksum_sha256 в накопительных счётчиках неверен"
            ) from exc

        stats = cls()
        stats.rows = rows
        stats.min_event_timestamp = payload.get("min_event_timestamp")
        stats.max_event_timestamp = payload.get("max_event_timestamp")
        stats._checksum_sum = checksum_sum
        return stats


class _LegacyTopicManifestStats(_TopicManifestStats):
    """Прежняя контрольная сумма для проверки неизменённого артефакта."""

    def __init__(self):
        super().__init__()
        self._legacy_checksum = hashlib.sha256()

    @property
    def checksum(self) -> str:
        return self._legacy_checksum.hexdigest()

    def add(self, event: dict, event_timestamp: str | None = None) -> None:
        self.rows += 1
        self._legacy_checksum.update(
            json.dumps(event, sort_keys=True, ensure_ascii=True).encode("utf-8")
        )
        timestamp = (
            event_timestamp
            if event_timestamp is not None
            else event.get("event_timestamp")
        )
        self.add_timestamp(timestamp)


class ManifestCounters:
    """Счётчики стартовой истории для manifest."""

    def __init__(self):
        self.topic_stats = {topic: _TopicManifestStats() for topic in TOPICS}
        self.click_ids: set[str] = set()
        self.user_ids: set[str] = set()
        self._new_click_ids: set[str] = set()
        self._new_user_ids: set[str] = set()
        self._visit_count = 0
        self._user_count = 0
        self._previous_id_set_chain = _empty_id_set_chain()

    def add_batch(self, batch: dict[str, list[dict]]) -> None:
        browser_events = batch.get("browser_events", [])
        event_timestamps = {
            event["event_id"]: event.get("event_timestamp")
            for event in browser_events
            if event.get("event_id") and event.get("event_timestamp")
        }
        click_timestamps: dict[str, list[str]] = {}
        for event in browser_events:
            click_id = event.get("click_id")
            timestamp = event.get("event_timestamp")
            if click_id and timestamp:
                click_timestamps.setdefault(click_id, []).append(timestamp)

        for topic, events in batch.items():
            if topic not in self.topic_stats:
                raise ValueError(f"unknown startup history topic: {topic}")
            stats = self.topic_stats[topic]
            for event in events:
                event_timestamp = event.get("event_timestamp")
                if event_timestamp is None and topic == "location_events":
                    event_timestamp = event_timestamps.get(event.get("event_id"))
                if event_timestamp is None and topic in ("device_events", "geo_events"):
                    timestamps = click_timestamps.get(event.get("click_id"))
                    if timestamps:
                        event_timestamp = min(timestamps)
                        stats.add_timestamp(max(timestamps))
                stats.add(event, event_timestamp=event_timestamp)
                click_id = event.get("click_id")
                if (
                    topic == "browser_events"
                    and click_id
                    and click_id not in self.click_ids
                ):
                    self.click_ids.add(click_id)
                    self._new_click_ids.add(click_id)
                    self._visit_count += 1
                user_id = event.get("user_domain_id")
                if (
                    topic == "device_events"
                    and user_id
                    and user_id not in self.user_ids
                ):
                    self.user_ids.add(user_id)
                    self._new_user_ids.add(user_id)
                    self._user_count += 1

    def to_manifest_topics(self) -> dict:
        return {topic: stats.to_dict() for topic, stats in self.topic_stats.items()}

    def to_manifest_totals(self) -> dict:
        browser_stats = self.topic_stats["browser_events"]
        return {
            "events": browser_stats.rows,
            "visits": self._visit_count,
            "users": self._user_count,
            "min_event_timestamp": browser_stats.min_event_timestamp,
            "max_event_timestamp": browser_stats.max_event_timestamp,
        }

    def to_state(self) -> dict:
        """Возвращает JSON-сериализуемое накопительное состояние."""
        state, _chunks = self.to_state_with_chunks()
        return state

    def to_state_with_chunks(self) -> tuple[dict, list[tuple[str, dict]]]:
        """Возвращает компактное состояние и новые фрагменты точных множеств."""
        id_set_chain, chunks = _extend_id_set_chain(
            self._previous_id_set_chain,
            self._new_click_ids,
            self._new_user_ids,
        )
        state = {
            "version": COUNTER_STATE_VERSION,
            "checksum_algorithm": "sha256-sum-v1",
            "topics": self.to_manifest_topics(),
            "totals": {
                "visits": self._visit_count,
                "users": self._user_count,
            },
            "id_sets": id_set_chain,
        }
        return state, chunks

    @classmethod
    def from_state(
        cls,
        payload: dict,
        *,
        known_click_ids: set[str] | None = None,
        known_user_ids: set[str] | None = None,
    ) -> "ManifestCounters":
        """Продолжает счётчики из state без чтения старых событий."""
        if not isinstance(payload, dict):
            raise ValueError("накопительные счётчики должны быть объектом")
        if payload.get("version") != COUNTER_STATE_VERSION:
            raise ValueError("версия накопительных счётчиков не поддерживается")
        if payload.get("checksum_algorithm") != "sha256-sum-v1":
            raise ValueError("алгоритм накопительной контрольной суммы не поддерживается")
        topic_payloads = payload.get("topics")
        if not isinstance(topic_payloads, dict) or set(topic_payloads) != set(TOPICS):
            raise ValueError("в накопительных счётчиках нет всех топиков")
        totals = payload.get("totals")
        if not isinstance(totals, dict):
            raise ValueError("в накопительных счётчиках нет итогов")
        visits = _non_negative_int(totals.get("visits"), "visits")
        users = _non_negative_int(totals.get("users"), "users")
        id_sets = payload.get("id_sets")
        _validate_id_set_chain(id_sets, visits=visits, users=users)

        counters = cls()
        counters.topic_stats = {
            topic: _TopicManifestStats.from_state(topic_payloads[topic])
            for topic in TOPICS
        }
        counters.click_ids = set(known_click_ids or ())
        counters.user_ids = set(known_user_ids or ())
        counters._new_click_ids = set()
        counters._new_user_ids = set()
        counters._visit_count = visits
        counters._user_count = users
        counters._previous_id_set_chain = dict(id_sets)
        return counters


def cumulative_counter_reference(counter_state: dict) -> dict:
    """Строит компактную ссылку state на полный набор из manifest."""
    canonical = json.dumps(
        counter_state,
        sort_keys=True,
        separators=(",", ":"),
        ensure_ascii=True,
    ).encode("utf-8")
    return {
        "version": COUNTER_STATE_VERSION,
        "manifest_sha256": hashlib.sha256(canonical).hexdigest(),
    }


def known_counter_ids_from_state(state: GeneratorState) -> tuple[set[str], set[str]]:
    """Возвращает идентификаторы, которые уже успели попасть в data-топики."""
    click_ids = {visit["click_id"] for visit in state.active_visits}
    user_ids = {
        user["user_domain_id"]
        for user in state.population
        if user.get("active_click_id") is not None
        or user.get("last_finished_at") is not None
    }
    return click_ids, user_ids


def _empty_id_set_chain() -> dict:
    return {
        "storage": ID_SET_STORAGE,
        "latest_sha256": None,
        "chunks": 0,
        "click_ids": 0,
        "user_ids": 0,
    }


def _non_negative_int(value: Any, label: str) -> int:
    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
        raise ValueError(f"{label} в накопительных счётчиках должен быть целым")
    return value


def _validate_id_set_chain(payload: Any, *, visits: int, users: int) -> None:
    if not isinstance(payload, dict):
        raise ValueError("в накопительных счётчиках нет множеств идентификаторов")
    if payload.get("storage") != ID_SET_STORAGE:
        raise ValueError("хранилище множеств идентификаторов не поддерживается")
    chunks = _non_negative_int(payload.get("chunks"), "chunks")
    click_ids = _non_negative_int(payload.get("click_ids"), "click_ids")
    user_ids = _non_negative_int(payload.get("user_ids"), "user_ids")
    latest = payload.get("latest_sha256")
    if latest is not None:
        if not isinstance(latest, str) or len(latest) != 64:
            raise ValueError("ссылка на фрагмент множеств идентификаторов неверна")
        try:
            int(latest, 16)
        except ValueError as exc:
            raise ValueError(
                "ссылка на фрагмент множеств идентификаторов неверна"
            ) from exc
    if (chunks == 0) != (latest is None):
        raise ValueError("цепочка множеств идентификаторов повреждена")
    if click_ids != visits or user_ids != users:
        raise ValueError("размеры множеств идентификаторов не совпадают с итогами")


def _extend_id_set_chain(
    previous: dict,
    click_ids: set[str],
    user_ids: set[str],
) -> tuple[dict, list[tuple[str, dict]]]:
    _validate_id_set_chain(
        previous,
        visits=_non_negative_int(previous.get("click_ids"), "click_ids"),
        users=_non_negative_int(previous.get("user_ids"), "user_ids"),
    )
    tagged_ids = [("click_id", value) for value in sorted(click_ids)]
    tagged_ids.extend(("user_id", value) for value in sorted(user_ids))
    previous_sha256 = previous["latest_sha256"]
    chunk_count = previous["chunks"]
    chunks: list[tuple[str, dict]] = []
    for offset in range(0, len(tagged_ids), ID_SET_CHUNK_SIZE):
        part = tagged_ids[offset : offset + ID_SET_CHUNK_SIZE]
        part_click_ids = {
            value for kind, value in part if kind == "click_id"
        }
        part_user_ids = {
            value for kind, value in part if kind == "user_id"
        }
        chunk = {
            "version": ID_SET_CHUNK_VERSION,
            "encoding": "json-zlib-base64-v1",
            "sequence": chunk_count + 1,
            "previous_sha256": previous_sha256,
            "click_ids": _encode_id_set(part_click_ids),
            "user_ids": _encode_id_set(part_user_ids),
        }
        digest = hashlib.sha256(_canonical_json(chunk)).hexdigest()
        chunks.append((digest, chunk))
        previous_sha256 = digest
        chunk_count += 1

    return (
        {
            "storage": ID_SET_STORAGE,
            "latest_sha256": previous_sha256,
            "chunks": chunk_count,
            "click_ids": previous["click_ids"] + len(click_ids),
            "user_ids": previous["user_ids"] + len(user_ids),
        },
        chunks,
    )


def _canonical_json(value: dict) -> bytes:
    return json.dumps(
        value,
        sort_keys=True,
        separators=(",", ":"),
        ensure_ascii=True,
    ).encode("utf-8")


def _encode_id_set(values: set[str]) -> str:
    raw = json.dumps(sorted(values), separators=(",", ":")).encode("utf-8")
    return base64.b64encode(zlib.compress(raw, level=9)).decode("ascii")


class _LegacyManifestCounters(ManifestCounters):
    """Полный пересчёт старой суммы только при проверке артефакта."""

    def __init__(self):
        super().__init__()
        self.topic_stats = {
            topic: _LegacyTopicManifestStats() for topic in TOPICS
        }


class StartupHistoryArtifactBuilder:
    """Собирает сообщения топиков для портативного файла."""

    def __init__(self):
        self.topics = {topic: [] for topic in TOPICS}
        self.raw_topics = {topic: [] for topic in TOPICS}
        self.counters = ManifestCounters()

    def add_batch(self, batch: dict[str, list[dict]]) -> None:
        self.counters.add_batch(batch)
        for topic in TOPICS:
            for event in batch.get(topic, []):
                self.topics[topic].append(deepcopy(event))
                self.raw_topics[topic].append(
                    {
                        "key": _event_key(event),
                        "value_json": _event_value_json(event),
                    }
                )

    def to_artifact(self, manifest: dict, state: GeneratorState) -> dict:
        artifact_manifest = deepcopy(manifest)
        artifact_manifest.pop("cumulative_manifest_counters", None)
        artifact_state = state.to_dict()
        artifact_state.pop("cumulative_manifest_counters", None)
        return {
            "artifact_version": ARTIFACT_VERSION,
            "kind": ARTIFACT_KIND,
            "created_at": datetime.now(timezone.utc).isoformat(),
            "manifest": artifact_manifest,
            "state": artifact_state,
            "topics": deepcopy(self.topics),
            "raw_topics": deepcopy(self.raw_topics),
        }


class KafkaRawPublisher:
    """Публикует сохранённые Kafka value bytes без повторной JSON-сериализации."""

    def __init__(self, bootstrap_servers: str):
        self.bootstrap_servers = bootstrap_servers
        KafkaProducerCls, _ = _kafka_importer()()
        self.producer = KafkaProducerCls(
            bootstrap_servers=self.bootstrap_servers,
            key_serializer=lambda k: k.encode("utf-8") if k else None,
            retries=3,
            retry_backoff_ms=1000,
        )

    def publish_records(self, topic: str, records: list[dict]) -> tuple[int, int]:
        sent = 0
        errors = 0
        futures = []
        for record in records:
            future = self.producer.send(
                topic,
                key=record.get("key"),
                value=record["value_json"].encode("utf-8"),
            )
            futures.append(future)
        for future in futures:
            try:
                future.get(timeout=10)
                sent += 1
            except Exception:
                errors += 1
        return sent, errors

    def flush(self) -> None:
        self.producer.flush()

    def close(self) -> None:
        self.producer.close()


class KafkaTopicInspector:
    """Проверяет Kafka-топики перед clean-stand импортом."""

    def __init__(self, bootstrap_servers: str):
        self.bootstrap_servers = bootstrap_servers

    def assert_data_topics_empty(self) -> None:
        """Падает, если в data-топиках уже есть сообщения."""
        from kafka import KafkaConsumer, TopicPartition

        consumer = KafkaConsumer(
            bootstrap_servers=self.bootstrap_servers,
            enable_auto_commit=False,
            consumer_timeout_ms=1000,
        )
        try:
            dirty_topics = []
            for topic in TOPICS:
                partitions = consumer.partitions_for_topic(topic)
                if not partitions:
                    continue
                topic_partitions = [
                    TopicPartition(topic, partition)
                    for partition in partitions
                ]
                end_offsets = consumer.end_offsets(topic_partitions)
                if any(offset > 0 for offset in end_offsets.values()):
                    dirty_topics.append(topic)
            if dirty_topics:
                raise RuntimeError(
                    "Kafka data topics are not empty; clean the stand before "
                    "startup history import: " + ", ".join(dirty_topics)
                )
        finally:
            consumer.close()

    def snapshot_import_topics(self) -> dict:
        """Снимок для отката clean-stand импорта."""
        return {"topics": list(IMPORT_TOPICS)}

    def rollback_import_topics(self, snapshot: dict) -> None:
        """Удаляет топики, которые могли получить частичный импорт."""
        from kafka import KafkaAdminClient
        from kafka.errors import UnknownTopicOrPartitionError

        admin = KafkaAdminClient(bootstrap_servers=self.bootstrap_servers)
        try:
            existing_topics = set(admin.list_topics())
            topics = [
                topic
                for topic in snapshot.get("topics", IMPORT_TOPICS)
                if topic in existing_topics
            ]
            if not topics:
                return
            try:
                admin.delete_topics(topics)
            except UnknownTopicOrPartitionError:
                return
        finally:
            admin.close()


def generation_settings_from_config(config: Config) -> dict:
    """Возвращает настройки, влияющие на поток генерации."""
    return {
        "tick_seconds": config.tick_seconds,
        "lambda_base_per_min": config.lambda_base_per_min,
        "jitter_pct": config.jitter_pct,
        "min_events_per_tick": config.min_events_per_tick,
        "max_events_per_tick": config.max_events_per_tick,
        "max_session_events": config.max_session_events,
        "max_active_sessions": config.max_active_sessions,
        "population_max": config.population_max,
        "p_new_user": config.p_new_user,
        "min_return_minutes": config.min_return_minutes,
        "model_time_speed": config.model_time_speed,
    }


def build_manifest(
    config: Config,
    counters: ManifestCounters,
    state: GeneratorState,
    *,
    model_t_end: datetime | None = None,
    boundaries: list[str] | None = None,
    cumulative_state: dict | None = None,
) -> dict:
    """Строит manifest стартовой истории."""
    selected_t_end = model_t_end or config.model_t_end
    if selected_t_end is None:
        raise ValueError("GEN_MODEL_T_END is required for startup history manifest")
    selected_boundaries = (
        boundaries
        if boundaries is not None
        else [config.model_t0.isoformat(), selected_t_end.isoformat()]
    )
    manifest = {
        "manifest_version": "1.0",
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "gen_seed": config.seed,
        "model_t0": config.model_t0.isoformat(),
        "model_t_end": selected_t_end.isoformat(),
        "boundaries": list(selected_boundaries),
        "model_timezone": config.model_timezone,
        "run_mode": "backfill",
        "launch_profile": config.launch_profile,
        "generation_settings": generation_settings_from_config(config),
        "state_version": state.version,
        "state": {
            "topic": KafkaStateManager.STATE_TOPIC,
            "key": KafkaStateManager.STATE_KEY,
            "last_batch_id": state.last_batch_id,
            "model_timestamp": state.model_timestamp.isoformat(),
        },
        "topics": counters.to_manifest_topics(),
        "totals": counters.to_manifest_totals(),
        "cumulative_manifest_counters": cumulative_state or counters.to_state(),
    }
    manifest_boundaries(manifest)
    return manifest


def manifest_boundaries(manifest: dict) -> list[str]:
    """Возвращает проверенную цепочку границ, включая старый формат."""
    model_t0 = _parse_manifest_timestamp(manifest.get("model_t0"))
    model_t_end = _parse_manifest_timestamp(manifest.get("model_t_end"))
    raw_boundaries = manifest.get("boundaries")
    if raw_boundaries is None:
        return [manifest["model_t0"], manifest["model_t_end"]]
    if not isinstance(raw_boundaries, list) or len(raw_boundaries) < 2:
        raise ValueError("manifest boundaries должен содержать T0 и T_end")

    parsed = [_parse_manifest_timestamp(value) for value in raw_boundaries]
    if parsed[0] != model_t0:
        raise ValueError("manifest boundaries не начинается с model_t0")
    if parsed[-1] != model_t_end:
        raise ValueError("manifest boundaries не заканчивается на model_t_end")
    if any(left >= right for left, right in zip(parsed, parsed[1:])):
        raise ValueError("manifest boundaries должен строго возрастать")
    return list(raw_boundaries)


def write_startup_history_artifact(path: str | Path, artifact: dict) -> None:
    """Пишет артефакт в JSON-файл."""
    target = Path(path)
    target.parent.mkdir(parents=True, exist_ok=True)
    if target.suffix == ".xz":
        with lzma.open(
            target,
            "wt",
            encoding="utf-8",
            preset=9 | lzma.PRESET_EXTREME,
        ) as output:
            json.dump(artifact, output, ensure_ascii=True, indent=2)
        return
    target.write_text(
        json.dumps(artifact, ensure_ascii=True, indent=2),
        encoding="utf-8",
    )


def load_startup_history_artifact(path: str | Path) -> dict:
    """Читает артефакт из JSON-файла."""
    source = Path(path)
    if source.suffix == ".xz":
        with lzma.open(source, "rt", encoding="utf-8") as input_file:
            return json.load(input_file)
    return json.loads(source.read_text(encoding="utf-8"))


def validate_startup_history_artifact(
    artifact: dict,
    expected_config: Config | None = None,
) -> tuple[GeneratorState, dict, dict[str, list[dict]], dict[str, list[dict]]]:
    """Проверяет связку manifest, state и сообщений топиков."""
    if not isinstance(artifact, dict):
        raise ValueError("artifact must be an object")
    if artifact.get("kind") != ARTIFACT_KIND:
        raise ValueError("artifact kind mismatch")
    if artifact.get("artifact_version") != ARTIFACT_VERSION:
        raise ValueError("artifact version mismatch")

    manifest = artifact.get("manifest")
    state_payload = artifact.get("state")
    topics = artifact.get("topics")
    raw_topics = artifact.get("raw_topics")
    if not isinstance(manifest, dict):
        raise ValueError("artifact manifest must be an object")
    if not isinstance(state_payload, dict):
        raise ValueError("artifact state must be an object")
    if not isinstance(topics, dict):
        raise ValueError("artifact topics must be an object")
    if not isinstance(raw_topics, dict):
        raise ValueError("artifact raw_topics must be an object")
    for topic in TOPICS:
        if not isinstance(topics.get(topic), list):
            raise ValueError(f"artifact topic {topic} must be a list")
        if not isinstance(raw_topics.get(topic), list):
            raise ValueError(f"artifact raw topic {topic} must be a list")

    state = GeneratorState.from_dict(state_payload)
    manifest_boundaries(manifest)
    _validate_manifest_state(manifest, state)
    _validate_manifest_topics(manifest, topics)
    _validate_raw_topics(topics, raw_topics)
    if expected_config is not None:
        _validate_manifest_config(manifest, expected_config)
    return (
        state,
        manifest,
        {topic: topics[topic] for topic in TOPICS},
        {topic: raw_topics[topic] for topic in TOPICS},
    )


def import_startup_history_artifact(
    artifact: dict,
    *,
    publisher,
    state_manager,
    manifest_manager,
    expected_config: Config | None = None,
    topic_inspector=None,
) -> dict:
    """Воспроизводит артефакт в Kafka и compact-топиках генератора."""
    state, manifest, topics, raw_topics = validate_startup_history_artifact(
        artifact,
        expected_config=expected_config,
    )
    counters = ManifestCounters()
    counters.add_batch(topics)
    cumulative_state, counter_chunks = counters.to_state_with_chunks()
    state.cumulative_manifest_counters = cumulative_counter_reference(
        cumulative_state
    )
    manifest = deepcopy(manifest)
    manifest["topics"] = counters.to_manifest_topics()
    manifest["totals"] = counters.to_manifest_totals()
    manifest["cumulative_manifest_counters"] = cumulative_state
    if topic_inspector is not None:
        topic_inspector.assert_data_topics_empty()
        snapshot = (
            topic_inspector.snapshot_import_topics()
            if hasattr(topic_inspector, "snapshot_import_topics")
            else None
        )
    else:
        snapshot = None

    if not hasattr(publisher, "publish_records"):
        raise TypeError(
            "startup history import requires publisher.publish_records "
            "for raw Kafka values"
        )

    try:
        sent_total = 0
        sent_by_topic = {}

        for topic in TOPICS:
            raw_records = raw_topics[topic]
            if not raw_records:
                sent_by_topic[topic] = 0
                continue
            sent, errors = publisher.publish_records(topic, raw_records)
            if errors:
                raise RuntimeError(
                    f"Startup history import failed for {topic}: sent={sent}, errors={errors}"
                )
            sent_total += sent
            sent_by_topic[topic] = sent

        publisher.flush()
        for chunk_sha256, chunk in counter_chunks:
            manifest_manager.save_counter_chunk(chunk_sha256, chunk)
        if counter_chunks:
            manifest_manager.flush()
        manifest_manager.save(manifest)
        manifest_manager.flush()
        state_manager.save(state)
        state_manager.flush()
    except Exception:
        if (
            topic_inspector is not None
            and snapshot is not None
            and hasattr(topic_inspector, "rollback_import_topics")
        ):
            topic_inspector.rollback_import_topics(snapshot)
        raise

    return {"events": sent_total, "topics": sent_by_topic}


def compare_clickhouse_stats_to_manifest(
    manifest: dict,
    stats: dict[str, str],
) -> list[str]:
    """Возвращает поля ClickHouse, которые не совпали с manifest."""
    totals = manifest.get("totals") or {}
    expected = {
        "events": str(totals.get("events")),
        "visits": str(totals.get("visits")),
        "users": str(totals.get("users")),
        "min_event_timestamp": str(totals.get("min_event_timestamp")),
        "max_event_timestamp": str(totals.get("max_event_timestamp")),
    }
    return [
        field
        for field, expected_value in expected.items()
        if str(stats.get(field)) != expected_value
    ]


def _validate_manifest_state(manifest: dict, state: GeneratorState) -> None:
    mismatches = []
    expected_state = manifest.get("state") or {}
    if manifest.get("run_mode") != "backfill":
        mismatches.append("run_mode")
    if manifest.get("state_version") != state.version:
        mismatches.append("state_version")
    if expected_state.get("last_batch_id") != state.last_batch_id:
        mismatches.append("state.last_batch_id")
    if expected_state.get("model_timestamp") != state.model_timestamp.isoformat():
        mismatches.append("state.model_timestamp")
    try:
        model_t0 = _parse_manifest_timestamp(manifest["model_t0"])
        model_t_end = _parse_manifest_timestamp(manifest["model_t_end"])
    except (KeyError, TypeError, ValueError) as exc:
        raise ValueError(f"manifest time fields are invalid: {exc}") from exc
    if state.model_t0 != model_t0:
        mismatches.append("GEN_MODEL_T0")
    if state.model_timestamp != model_t_end:
        mismatches.append("GEN_MODEL_T_END")
    if state.gen_seed != manifest.get("gen_seed"):
        mismatches.append("GEN_SEED")
    if state.model_timezone != manifest.get("model_timezone"):
        mismatches.append("GEN_MODEL_TIMEZONE")
    settings = manifest.get("generation_settings") or {}
    if abs(state.model_time_speed - settings.get("model_time_speed", -1)) > 1e-9:
        mismatches.append("GEN_MODEL_TIME_SPEED")
    if mismatches:
        raise ValueError("startup history artifact mismatch: " + ", ".join(mismatches))


def validate_manifest_state(manifest: dict, state: GeneratorState) -> None:
    """Проверяет согласованность manifest и state без данных артефакта."""
    _validate_manifest_state(manifest, state)


def _validate_manifest_topics(manifest: dict, topics: dict[str, list[dict]]) -> None:
    counters = ManifestCounters()
    counters.add_batch({topic: topics[topic] for topic in TOPICS})
    expected_topics = manifest.get("topics") or {}
    expected_totals = manifest.get("totals") or {}
    if counters.to_manifest_totals() != expected_totals:
        raise ValueError("startup history artifact mismatch: totals")
    if counters.to_manifest_topics() == expected_topics:
        return

    legacy_counters = _LegacyManifestCounters()
    legacy_counters.add_batch({topic: topics[topic] for topic in TOPICS})
    if legacy_counters.to_manifest_topics() != expected_topics:
        raise ValueError("startup history artifact mismatch: topic checksums")


def _validate_raw_topics(
    topics: dict[str, list[dict]],
    raw_topics: dict[str, list[dict]],
) -> None:
    for topic in TOPICS:
        if len(raw_topics[topic]) != len(topics[topic]):
            raise ValueError(f"artifact raw topic {topic} row count mismatch")
        for event, record in zip(topics[topic], raw_topics[topic]):
            if record.get("key") != _event_key(event):
                raise ValueError(f"artifact raw topic {topic} key mismatch")
            if _raw_value_to_event(record.get("value_json"), topic) != event:
                raise ValueError(f"artifact raw topic {topic} value mismatch")


def _event_key(event: dict) -> str | None:
    return event.get("event_id") or event.get("click_id")


def _event_value_json(event: dict) -> str:
    return kafka_event_value_json(event)


def _raw_value_to_event(value: Any, topic: str) -> dict:
    if not isinstance(value, str):
        raise ValueError(f"artifact raw topic {topic} value must be a string")
    try:
        event = json.loads(value)
    except json.JSONDecodeError as exc:
        raise ValueError(f"artifact raw topic {topic} value must be valid JSON") from exc
    if not isinstance(event, dict):
        raise ValueError(f"artifact raw topic {topic} value must be a JSON object")
    return event


def _validate_manifest_config(manifest: dict, config: Config) -> None:
    mismatches = []
    if manifest.get("gen_seed") != config.seed:
        mismatches.append("GEN_SEED")
    if _parse_manifest_timestamp(manifest.get("model_t0")) != config.model_t0:
        mismatches.append("GEN_MODEL_T0")
    if config.model_t_end is None:
        mismatches.append("GEN_MODEL_T_END")
    elif _parse_manifest_timestamp(manifest.get("model_t_end")) != config.model_t_end:
        mismatches.append("GEN_MODEL_T_END")
    if manifest.get("model_timezone") != config.model_timezone:
        mismatches.append("GEN_MODEL_TIMEZONE")
    if manifest.get("generation_settings") != generation_settings_from_config(config):
        mismatches.append("generation_settings")
    if mismatches:
        raise ValueError(
            "startup history artifact is incompatible with current settings: "
            + ", ".join(mismatches)
        )


def _parse_manifest_timestamp(value: Any) -> datetime:
    if not isinstance(value, str):
        raise ValueError("timestamp must be a string")
    timestamp = datetime.fromisoformat(value.replace("Z", "+00:00"))
    if timestamp.tzinfo is None:
        raise ValueError("timestamp must include timezone")
    return timestamp.astimezone(timezone.utc)


def _run_import(args: argparse.Namespace) -> int:
    config = Config()
    artifact = load_startup_history_artifact(args.artifact)
    ensure_topics(config.kafka_bootstrap_servers)
    publisher = KafkaRawPublisher(config.kafka_bootstrap_servers)
    state_manager = KafkaStateManager(config.kafka_bootstrap_servers)
    manifest_manager = KafkaStartupHistoryManifest(config.kafka_bootstrap_servers)
    topic_inspector = KafkaTopicInspector(config.kafka_bootstrap_servers)
    try:
        result = import_startup_history_artifact(
            artifact,
            publisher=publisher,
            state_manager=state_manager,
            manifest_manager=manifest_manager,
            expected_config=config,
            topic_inspector=topic_inspector,
        )
    finally:
        publisher.close()
        state_manager.close()
        manifest_manager.close()
    logger.info(
        "Startup history artifact imported: events=%s, topics=%s",
        result["events"],
        result["topics"],
    )
    return 0


def _run_validate(args: argparse.Namespace) -> int:
    config = Config()
    artifact = load_startup_history_artifact(args.artifact)
    validate_startup_history_artifact(artifact, expected_config=config)
    logger.info("Startup history artifact is valid: %s", args.artifact)
    return 0


def _run_manifest_summary(args: argparse.Namespace) -> int:
    artifact = load_startup_history_artifact(args.artifact)
    _, manifest, _, _ = validate_startup_history_artifact(artifact)
    totals = manifest["totals"]
    print(
        "\t".join(
            [
                str(totals["events"]),
                str(totals["visits"]),
                str(totals["users"]),
                str(totals["min_event_timestamp"]),
                str(totals["max_event_timestamp"]),
                str(manifest["model_t0"]),
                str(manifest["model_t_end"]),
            ]
        )
    )
    return 0


def _print_manifest_summary(manifest: dict) -> None:
    totals = manifest["totals"]
    print(
        "\t".join(
            [
                str(totals["events"]),
                str(totals["visits"]),
                str(totals["users"]),
                str(totals["min_event_timestamp"]),
                str(totals["max_event_timestamp"]),
                str(manifest["model_t0"]),
                str(manifest["model_t_end"]),
                str(manifest.get("launch_profile", "")),
            ]
        )
    )


def _run_kafka_manifest_summary(args: argparse.Namespace) -> int:
    config = Config()
    manifest_manager = KafkaStartupHistoryManifest(config.kafka_bootstrap_servers)
    try:
        manifest = manifest_manager.load()
    finally:
        manifest_manager.close()

    if not manifest:
        raise RuntimeError("startup history manifest not found in Kafka")

    _print_manifest_summary(manifest)
    return 0


def _run_kafka_boundaries(args: argparse.Namespace) -> int:
    config = Config()
    manifest_manager = KafkaStartupHistoryManifest(config.kafka_bootstrap_servers)
    try:
        manifest = manifest_manager.load()
    finally:
        manifest_manager.close()
    if not manifest:
        raise RuntimeError("manifest стартовой истории не найден в Kafka")
    for boundary in manifest_boundaries(manifest):
        print(boundary)
    return 0


def _run_kafka_topic_rows(args: argparse.Namespace) -> int:
    config = Config()
    manifest_manager = KafkaStartupHistoryManifest(config.kafka_bootstrap_servers)
    try:
        manifest = manifest_manager.load()
    finally:
        manifest_manager.close()
    if not manifest:
        raise RuntimeError("manifest стартовой истории не найден в Kafka")
    topics = manifest.get("topics") or {}
    print("\t".join(str(topics[topic]["rows"]) for topic in TOPICS))
    return 0


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description="Startup history artifact tools")
    subparsers = parser.add_subparsers(dest="command", required=True)

    validate_parser = subparsers.add_parser("validate")
    validate_parser.add_argument("--artifact", required=True)
    validate_parser.set_defaults(func=_run_validate)

    import_parser = subparsers.add_parser("import")
    import_parser.add_argument("--artifact", required=True)
    import_parser.set_defaults(func=_run_import)

    summary_parser = subparsers.add_parser("manifest-summary")
    summary_parser.add_argument("--artifact", required=True)
    summary_parser.set_defaults(func=_run_manifest_summary)

    kafka_summary_parser = subparsers.add_parser("kafka-manifest-summary")
    kafka_summary_parser.set_defaults(func=_run_kafka_manifest_summary)

    kafka_boundaries_parser = subparsers.add_parser("kafka-boundaries")
    kafka_boundaries_parser.set_defaults(func=_run_kafka_boundaries)

    kafka_topic_rows_parser = subparsers.add_parser("kafka-topic-rows")
    kafka_topic_rows_parser.set_defaults(func=_run_kafka_topic_rows)

    args = parser.parse_args(argv)
    return args.func(args)


if __name__ == "__main__":
    raise SystemExit(main())
