ddmitry/clickstream-ch-kafka-superset-demo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(generator): инкрементальные счётчики manifest без перечитки Kafka

Browse Source 
- Зачем:
  - world_next_day перечитывал всю историю топиков Kafka ради
    накопительных счётчиков — время прогона росло с возрастом мира
    (issue #5, находка F9).
- Что:
  - счётчики засеваются при import из уже прочитанного артефакта и при
    backfill из потока; next-day продвигает их только событиями нового
    дня, полного чтения Kafka больше нет;
  - катящаяся контрольная сумма — сумма SHA-256 событий по модулю 2^256
    (инкремент равен полному пересчёту), старый формат артефакта
    принимается без изменений;
  - точные множества click_id/user_domain_id вынесены из manifest в
    цепочку контент-адресуемых фрагментов (<=10 000 ID, SHA-256-цепочка,
    отдельный топик counter_chunks) — потолок сообщения Kafka не грозит,
    предел 900 000 байт проверяется явно с понятной ошибкой;
  - порядок записи всюду: фрагменты -> manifest -> state; старое локальное
    состояние отклоняется с подсказкой перезапустить import;
  - документация manifest/state обновлена (ARCHITECTURE, OPERATIONS,
    runbook startup-history).
- Проверка:
  - make test (216+31) и make lint зелёные;
  - живая приёмка на чистом стенде: import 235 с; три прогона
    world_next_day — 716/718/716 с (плоское время, O(нового дня));
    мир 3->6 дней, 561 942 события; make generated-history-chain-check —
    все порции и стыки однородны;
  - тест равенства инкремента и полного пересчёта:
    test_incremental_counters_equal_full_recompute.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
ddadmin
2026-07-22 23:47:02 +03:00
co-authored by Claude Fable 5
parent f4971e94ca
commit 103ac021c8
12 changed files with 845 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/tests/test_airflow_control.py
+5
View File
@@ -149,6 +149,11 @@ def test_next_day_precheck_requires_manifest_and_matching_state():
with pytest.raises(RuntimeError, match="state.*T_end"):
assert_next_day_snapshot(manifest, state)
manifest["model_t_end"] = state.model_timestamp.isoformat()
manifest["state"]["model_timestamp"] = state.model_timestamp.isoformat()
with pytest.raises(RuntimeError, match="повторно запустите import"):
assert_next_day_snapshot(manifest, state)
def test_target_dag_trigger_error_covers_missing_paused_and_ready_states():
"""Проверка зависимого DAG различает три реальные ветки."""
generator/tests/test_service.py
+97 -14
View File
@@ -25,6 +25,7 @@ from generator import (
main,
)
from clickstream_generator.state import UnsupportedStateVersionError
from clickstream_generator.service import IncompatibleStateError
class TestBatchRecordWithDictConversion:
@@ -425,6 +426,10 @@ class TestGeneratorServiceBackfill:
self, base_config
):
"""Backfill пишет [T0, T_end), state на T_end и повторяемый manifest."""
from clickstream_generator.startup_history_artifact import (
cumulative_counter_reference,
)
model_t0 = datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc)
model_t_end = model_t0 + timedelta(minutes=3)
config = replace(
@@ -463,6 +468,11 @@ class TestGeneratorServiceBackfill:
assert manifest["model_t0"] == model_t0.isoformat()
assert manifest["model_t_end"] == model_t_end.isoformat()
assert manifest["state"]["last_batch_id"] == saved_state.last_batch_id
assert saved_state.cumulative_manifest_counters == (
cumulative_counter_reference(
manifest["cumulative_manifest_counters"]
)
)
assert manifest["topics"]["browser_events"]["rows"] == len(browser_events)
for topic in (
"browser_events",
@@ -516,6 +526,8 @@ class TestGeneratorServiceBackfill:
self._run_backfill(config)
artifact = load_startup_history_artifact(artifact_path)
assert "cumulative_manifest_counters" not in artifact["state"]
assert "cumulative_manifest_counters" not in artifact["manifest"]
state, manifest, topics, _ = validate_startup_history_artifact(
artifact,
expected_config=config,
@@ -960,7 +972,8 @@ class TestGeneratorServiceBackfill:
with pytest.raises(RuntimeError, match="flush down"):
service._run_backfill()
service.manifest_manager.save.assert_called_once()
service.manifest_manager.save_counter_chunk.assert_called()
service.manifest_manager.save.assert_not_called()
service.manifest_manager.flush.assert_called_once()
service.state_manager.save.assert_not_called()
service.state_manager.flush.assert_not_called()
@@ -1008,13 +1021,14 @@ class TestGeneratorServiceBackfill:
class TestGeneratorServiceNextDay:
"""Проверки ограниченной доливки следующего модельного дня."""
def test_next_day_publishes_24h_then_state_then_cumulative_manifest(
def test_next_day_publishes_24h_then_chunks_manifest_and_state(
self, base_config
):
"""Next-day пишет [T_end, T_end+24h), затем state и manifest."""
"""Next-day пишет день, фрагменты множеств, manifest и затем state."""
from clickstream_generator.startup_history_artifact import (
StartupHistoryArtifactBuilder,
build_manifest,
cumulative_counter_reference,
)
model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
@@ -1062,6 +1076,9 @@ class TestGeneratorServiceNextDay:
}
initial_builder = StartupHistoryArtifactBuilder()
initial_builder.add_batch(old_batch)
state.cumulative_manifest_counters = cumulative_counter_reference(
initial_builder.counters.to_state()
)
manifest = build_manifest(config, initial_builder.counters, state)
published = {topic: [] for topic in old_batch}
@@ -1078,21 +1095,21 @@ class TestGeneratorServiceNextDay:
service.state_manager = MagicMock()
service.state_manager.save.side_effect = lambda _state: order.append("state")
service.manifest_manager = MagicMock()
service.manifest_manager.save_counter_chunk.side_effect = (
lambda _chunk_sha256, _chunk: order.append("counter_chunk")
)
service.manifest_manager.save.side_effect = (
lambda _manifest: order.append("manifest")
)
class DataReader:
def load(self):
return {
topic: old_batch[topic] + published[topic]
for topic in old_batch
}
raise AssertionError("next-day не должен перечитывать Kafka")
service.data_reader = DataReader()
service.restore_from_startup_history(state, model_t_end=current_t_end)
service._run_next_day(manifest)
service._run_next_day(manifest, state)
browser_events = published["browser_events"]
timestamps = [
@@ -1114,18 +1131,21 @@ class TestGeneratorServiceNextDay:
target_t_end.isoformat(),
]
assert saved_manifest["totals"]["events"] == len(browser_events) + 1
assert order == ["data", "state", "manifest"]
assert order == ["data", "counter_chunk", "manifest", "state"]
first_day_events = len(browser_events)
first_day_checksum = saved_manifest["topics"]["browser_events"][
"checksum_sha256"
]
service._run_next_day(saved_manifest)
service._run_next_day(saved_manifest, saved_state)
second_state = service.state_manager.save.call_args.args[0]
second_manifest = service.manifest_manager.save.call_args.args[0]
expected_builder = StartupHistoryArtifactBuilder()
expected_builder.add_batch(DataReader().load())
expected_builder.add_batch({
topic: old_batch[topic] + published[topic]
for topic in old_batch
})
assert second_state.model_timestamp == target_t_end + timedelta(hours=24)
assert second_manifest["boundaries"] == [
model_t0.isoformat(),
@@ -1140,15 +1160,57 @@ class TestGeneratorServiceNextDay:
!= first_day_checksum
)
assert second_manifest["topics"] == expected_builder.counters.to_manifest_topics()
assert second_manifest["totals"] == expected_builder.counters.to_manifest_totals()
assert second_state.cumulative_manifest_counters == (
cumulative_counter_reference(
second_manifest["cumulative_manifest_counters"]
)
)
assert order == [
"data",
"state",
"counter_chunk",
"manifest",
"state",
"data",
"state",
"counter_chunk",
"manifest",
"state",
]
def test_next_day_requires_cumulative_counters_from_import(self, base_config):
"""Старый локальный state требует повторного импорта, а не чтения Kafka."""
model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
config = replace(
base_config,
run_mode="next-day",
model_t0=model_t0,
model_t_end=model_t0 + timedelta(hours=1),
)
service = GeneratorService(config)
state = service.stream.to_state(
tick=1,
rng_state=service.generator.rng.getstate(),
last_batch_id="startup-history-old",
last_timestamp=config.model_t_end,
model_timestamp=config.model_t_end,
model_t0=config.model_t0,
)
service.publisher = MagicMock()
service.history = MagicMock()
service.state_manager = MagicMock()
service.manifest_manager = MagicMock()
service._model_time = config.model_t_end
manifest = {
"model_t0": model_t0.isoformat(),
"model_t_end": config.model_t_end.isoformat(),
"boundaries": [model_t0.isoformat(), config.model_t_end.isoformat()],
}
with pytest.raises(IncompatibleStateError, match="повторно запустите import"):
service._run_next_day(manifest, state)
service.publisher.publish.assert_not_called()
def test_next_day_publish_error_does_not_move_state_or_manifest(self, base_config):
"""Ошибка data-топика оставляет обе точки фиксации без изменений."""
model_t0 = datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc)
@@ -1177,8 +1239,29 @@ class TestGeneratorServiceNextDay:
"boundaries": [model_t0.isoformat(), config.model_t_end.isoformat()],
}
from clickstream_generator.startup_history_artifact import (
ManifestCounters,
cumulative_counter_reference,
)
state = service.stream.to_state(
tick=1,
rng_state=service.generator.rng.getstate(),
last_batch_id="startup-history-test",
last_timestamp=config.model_t_end,
model_timestamp=config.model_t_end,
model_t0=config.model_t0,
)
empty_counters = ManifestCounters()
manifest["cumulative_manifest_counters"] = empty_counters.to_state()
manifest["topics"] = empty_counters.to_manifest_topics()
manifest["totals"] = empty_counters.to_manifest_totals()
state.cumulative_manifest_counters = cumulative_counter_reference(
manifest["cumulative_manifest_counters"]
)
with pytest.raises(RuntimeError, match="Публикация next-day не удалась"):
service._run_next_day(manifest)
service._run_next_day(manifest, state)
service.state_manager.save.assert_not_called()
service.manifest_manager.save.assert_not_called()
generator/tests/test_startup_history_artifact.py
+210
View File
@@ -4,6 +4,7 @@
from dataclasses import replace
from datetime import datetime, timezone
import hashlib
import json
import lzma
@@ -110,6 +111,144 @@ def _batch() -> dict[str, list[dict]]:
}
def test_incremental_counters_equal_full_recompute():
"""Накопление по частям даёт те же числа, что полный пересчёт мира."""
from clickstream_generator.startup_history_artifact import ManifestCounters
first_day = _batch()
second_day = {
"browser_events": [
{
"event_id": "event-2",
"click_id": "click-1",
"user_domain_id": "user-1",
"event_timestamp": "2026-01-02 00:00:00.000000",
},
{
"event_id": "event-3",
"click_id": "click-2",
"user_domain_id": "user-2",
"event_timestamp": "2026-01-02 00:01:00.000000",
},
],
"location_events": [
{"event_id": "event-2", "page_url_path": "/cart"},
{"event_id": "event-3", "page_url_path": "/home"},
],
"device_events": [
{"click_id": "click-1", "user_domain_id": "user-1"},
{"click_id": "click-2", "user_domain_id": "user-2"},
],
"geo_events": [
{"click_id": "click-1", "country": "RU"},
{"click_id": "click-2", "country": "KZ"},
],
}
incremental = ManifestCounters()
incremental.add_batch(first_day)
incremental = ManifestCounters.from_state(
incremental.to_state(),
known_click_ids={"click-1"},
known_user_ids={"user-1"},
)
incremental.add_batch(second_day)
full = ManifestCounters()
full.add_batch({
topic: first_day[topic] + second_day[topic]
for topic in first_day
})
assert incremental.to_manifest_topics() == full.to_manifest_topics()
assert incremental.to_manifest_totals() == full.to_manifest_totals()
def test_exact_id_sets_are_split_into_bounded_hash_chain(monkeypatch):
"""Manifest ссылается на цепочку малых точных фрагментов, а не растёт сам."""
from clickstream_generator import startup_history_artifact as module
monkeypatch.setattr(module, "ID_SET_CHUNK_SIZE", 2)
counters = module.ManifestCounters()
batch = _batch()
batch["browser_events"].extend([
{
"event_id": "event-2",
"click_id": "click-2",
"event_timestamp": "2026-01-01 00:01:00.000000",
},
{
"event_id": "event-3",
"click_id": "click-3",
"event_timestamp": "2026-01-01 00:02:00.000000",
},
])
counters.add_batch(batch)
state, chunks = counters.to_state_with_chunks()
assert len(chunks) == 2
assert state["totals"] == {"visits": 3, "users": 1}
assert state["id_sets"] == {
"storage": "manifest-topic-sha256-chain-v1",
"latest_sha256": chunks[-1][0],
"chunks": 2,
"click_ids": 3,
"user_ids": 1,
}
assert chunks[0][1]["previous_sha256"] is None
assert chunks[1][1]["previous_sha256"] == chunks[0][0]
assert "click-1" not in json.dumps(state)
restored = module.ManifestCounters.from_state(
state,
known_click_ids={"click-3"},
known_user_ids={"user-1"},
)
restored.add_batch({
"browser_events": [
{
"event_id": "event-4",
"click_id": "click-3",
"event_timestamp": "2026-01-02 00:00:00.000000",
},
{
"event_id": "event-5",
"click_id": "click-4",
"event_timestamp": "2026-01-02 00:01:00.000000",
},
],
"location_events": [],
"device_events": [{"click_id": "click-4", "user_domain_id": "user-2"}],
"geo_events": [],
})
next_state, next_chunks = restored.to_state_with_chunks()
assert next_state["totals"] == {"visits": 4, "users": 2}
assert len(next_chunks) == 1
assert next_chunks[0][1]["previous_sha256"] == chunks[-1][0]
assert next_state["id_sets"]["chunks"] == 3
def test_counter_chunk_has_explicit_kafka_size_and_hash_guards():
"""Большой или неверно адресованный фрагмент падает до отправки в Kafka."""
from clickstream_generator.kafka_io import (
MAX_COMPACT_MESSAGE_BYTES,
KafkaStartupHistoryManifest,
)
manager = object.__new__(KafkaStartupHistoryManifest)
with pytest.raises(ValueError, match="безопасный предел"):
manager._assert_message_size(
{"payload": "x" * MAX_COMPACT_MESSAGE_BYTES},
"фрагмент",
)
with pytest.raises(ValueError, match="хеш.*не совпадает"):
manager.save_counter_chunk("0" * 64, {"version": "1.0"})
assert manager.COUNTER_TOPIC != manager.MANIFEST_TOPIC
def test_artifact_roundtrip_keeps_events_state_and_manifest(base_config):
"""Артефакт хранит события, state и manifest одним проверяемым набором."""
from clickstream_generator.startup_history_artifact import (
@@ -173,6 +312,43 @@ def test_artifact_roundtrip_keeps_events_state_and_manifest(base_config):
path.unlink(missing_ok=True)
def test_legacy_artifact_checksum_remains_valid(base_config):
"""Эталонный артефакт со старой суммой не требует пересборки."""
from clickstream_generator.startup_history_artifact import (
StartupHistoryArtifactBuilder,
build_manifest,
validate_startup_history_artifact,
)
state = _state()
config = replace(
base_config,
model_t0=state.model_t0,
model_t_end=state.model_timestamp,
model_time_speed=1,
model_timezone="UTC",
seed=42,
)
builder = StartupHistoryArtifactBuilder()
builder.add_batch(_batch())
manifest = build_manifest(config, builder.counters, state)
for topic, events in builder.topics.items():
checksum = hashlib.sha256()
for event in events:
checksum.update(
json.dumps(event, sort_keys=True, ensure_ascii=True).encode("utf-8")
)
manifest["topics"][topic]["checksum_sha256"] = checksum.hexdigest()
artifact = builder.to_artifact(manifest=manifest, state=state)
_, loaded_manifest, _, _ = validate_startup_history_artifact(
artifact,
expected_config=config,
)
assert loaded_manifest["topics"] == artifact["manifest"]["topics"]
def test_manifest_and_artifact_show_launch_profile(base_config):
"""Manifest и файл артефакта показывают выбранный профиль запуска."""
from clickstream_generator.startup_history_artifact import (
@@ -372,6 +548,8 @@ def test_raw_topic_value_can_keep_original_json_bytes(base_config):
manifest=build_manifest(config=config, counters=builder.counters, state=state),
state=state,
)
assert "cumulative_manifest_counters" not in artifact["state"]
assert "cumulative_manifest_counters" not in artifact["manifest"]
raw_value = (
'{"event_timestamp":"2026-01-01 00:00:00.000000",'
'"user_domain_id":"user-1","click_id":"click-1","event_id":"event-1"}'
@@ -391,9 +569,15 @@ def test_raw_topic_value_can_keep_original_json_bytes(base_config):
return None
class CompactWriter:
def __init__(self):
self.chunks = []
def save(self, value):
self.saved = value
def save_counter_chunk(self, chunk_sha256, chunk):
self.chunks.append((chunk_sha256, chunk))
def flush(self):
return None
@@ -609,8 +793,10 @@ def test_import_rolls_back_kafka_topics_after_partial_publish(base_config):
def test_import_replays_events_and_compact_topics(base_config):
"""Импорт пишет события в Kafka и служебные compact-топики, не трогая ClickHouse."""
from clickstream_generator.startup_history_artifact import (
ManifestCounters,
StartupHistoryArtifactBuilder,
build_manifest,
cumulative_counter_reference,
import_startup_history_artifact,
)
@@ -646,10 +832,14 @@ def test_import_replays_events_and_compact_topics(base_config):
def __init__(self):
self.saved = None
self.flushed = False
self.chunks = []
def save(self, value):
self.saved = value
def save_counter_chunk(self, chunk_sha256, chunk):
self.chunks.append((chunk_sha256, chunk))
def flush(self):
self.flushed = True
@@ -678,7 +868,27 @@ def test_import_replays_events_and_compact_topics(base_config):
)
assert result["events"] == 4
assert state_manager.saved.last_batch_id == state.last_batch_id
assert state_manager.saved.cumulative_manifest_counters is not None
assert manifest_manager.saved["state"]["last_batch_id"] == state.last_batch_id
assert state_manager.saved.cumulative_manifest_counters == (
cumulative_counter_reference(
manifest_manager.saved["cumulative_manifest_counters"]
)
)
seeded = ManifestCounters.from_state(
manifest_manager.saved["cumulative_manifest_counters"],
known_click_ids={"click-1"},
known_user_ids={"user-1"},
)
assert seeded.click_ids == {"click-1"}
assert seeded.user_ids == {"user-1"}
assert len(manifest_manager.chunks) == 1
assert (
manifest_manager.saved["cumulative_manifest_counters"]["id_sets"][
"latest_sha256"
]
== manifest_manager.chunks[0][0]
)
assert publisher.flushed
assert state_manager.flushed
assert manifest_manager.flushed
generator/tests/test_state.py
+19
View File
@@ -227,6 +227,25 @@ class TestGeneratorState:
assert next_values == values_after
def test_state_roundtrip_keeps_cumulative_manifest_counters(self):
"""State сохраняет накопительные числа для следующего дня."""
counter_reference = {
"version": "1.0",
"manifest_sha256": "a" * 64,
}
state = GeneratorState(
tick=1,
rng_state=_make_valid_rng_state(),
last_batch_id="startup-history-test",
last_timestamp=datetime.now(timezone.utc),
population=_minimal_population(),
cumulative_manifest_counters=counter_reference,
)
restored = GeneratorState.from_dict(json.loads(json.dumps(state.to_dict())))
assert restored.cumulative_manifest_counters == counter_reference
def test_state_roundtrip_keeps_population_and_active_visits(self):
"""State хранит популяцию и активные визиты в JSON."""
rng_state = _make_valid_rng_state(42)