feat(generator): добавлен артефакт стартовой истории
- Зачем: - чистый стенд должен восстанавливать стартовую историю без повторной генерации. - Что: - добавлены export/import артефакта через Kafka и compact-топики. - добавлена manifest-aware сверка ClickHouse и защита от смешения state. - добавлен runbook использования стартовой истории. - Проверка: - uv run --with-requirements generator/requirements.txt pytest generator/tests -q. - bash -n scripts/check_startup_history_manifest.sh scripts/export_startup_history_artifact.sh scripts/import_startup_history_artifact.sh. - git diff --check.
This commit is contained in:
@@ -0,0 +1,518 @@
|
||||
"""
|
||||
Тесты портативного артефакта стартовой истории.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from generator import GeneratorState
|
||||
|
||||
|
||||
def _state() -> GeneratorState:
|
||||
import random
|
||||
|
||||
now = datetime(2026, 1, 1, 1, 0, tzinfo=timezone.utc)
|
||||
return GeneratorState(
|
||||
tick=1,
|
||||
rng_state=random.Random(42).getstate(),
|
||||
last_batch_id="startup-history-abc",
|
||||
last_timestamp=now,
|
||||
model_timestamp=now,
|
||||
wall_timestamp=now,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
model_t0=datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc),
|
||||
gen_seed=42,
|
||||
population=[
|
||||
{
|
||||
"user_domain_id": "user-1",
|
||||
"seed_click_id": "seed-1",
|
||||
"active_click_id": None,
|
||||
"last_finished_at": None,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _batch() -> dict[str, list[dict]]:
|
||||
return {
|
||||
"browser_events": [
|
||||
{
|
||||
"event_id": "event-1",
|
||||
"click_id": "click-1",
|
||||
"user_domain_id": "user-1",
|
||||
"event_timestamp": "2026-01-01 00:00:00.000000",
|
||||
}
|
||||
],
|
||||
"location_events": [
|
||||
{
|
||||
"event_id": "event-1",
|
||||
"page_url_path": "/home",
|
||||
}
|
||||
],
|
||||
"device_events": [
|
||||
{
|
||||
"click_id": "click-1",
|
||||
"user_domain_id": "user-1",
|
||||
}
|
||||
],
|
||||
"geo_events": [
|
||||
{
|
||||
"click_id": "click-1",
|
||||
"country": "RU",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_artifact_roundtrip_keeps_events_state_and_manifest(base_config):
|
||||
"""Артефакт хранит события, state и manifest одним проверяемым набором."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
load_startup_history_artifact,
|
||||
validate_startup_history_artifact,
|
||||
write_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
manifest = build_manifest(
|
||||
config=replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
),
|
||||
counters=builder.counters,
|
||||
state=state,
|
||||
)
|
||||
|
||||
artifact = builder.to_artifact(manifest=manifest, state=state)
|
||||
(
|
||||
loaded_state,
|
||||
loaded_manifest,
|
||||
loaded_topics,
|
||||
loaded_raw_topics,
|
||||
) = validate_startup_history_artifact(
|
||||
artifact,
|
||||
expected_config=replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
),
|
||||
)
|
||||
|
||||
assert artifact["artifact_version"] == "1.0"
|
||||
assert loaded_state.last_batch_id == state.last_batch_id
|
||||
assert loaded_manifest["state"]["last_batch_id"] == state.last_batch_id
|
||||
assert loaded_topics["browser_events"][0]["event_id"] == "event-1"
|
||||
assert loaded_raw_topics["browser_events"][0]["value_json"].encode("utf-8") == (
|
||||
b'{"event_id": "event-1", "click_id": "click-1", '
|
||||
b'"user_domain_id": "user-1", '
|
||||
b'"event_timestamp": "2026-01-01 00:00:00.000000"}'
|
||||
)
|
||||
|
||||
path = base_config.data_dir.parent / "tmp-startup-history-artifact.json"
|
||||
try:
|
||||
write_startup_history_artifact(path, artifact)
|
||||
loaded = load_startup_history_artifact(path)
|
||||
assert loaded["manifest"]["totals"]["events"] == 1
|
||||
finally:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def test_artifact_validation_rejects_mismatched_manifest(base_config):
|
||||
"""Валидация отвергает артефакт, где manifest не совпадает с событиями."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
validate_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
config = replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
)
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
manifest = build_manifest(config=config, counters=builder.counters, state=state)
|
||||
artifact = builder.to_artifact(manifest=manifest, state=state)
|
||||
artifact["manifest"]["gen_seed"] = 43
|
||||
|
||||
with pytest.raises(ValueError, match="GEN_SEED"):
|
||||
validate_startup_history_artifact(artifact, expected_config=config)
|
||||
|
||||
|
||||
def test_manifest_clickhouse_stats_detect_mismatch():
|
||||
"""Сверка ClickHouse с manifest ловит расхождение контрольных чисел."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
compare_clickhouse_stats_to_manifest,
|
||||
)
|
||||
|
||||
manifest = {
|
||||
"totals": {
|
||||
"events": 10,
|
||||
"visits": 4,
|
||||
"users": 3,
|
||||
"min_event_timestamp": "2026-01-01 00:00:00.000000",
|
||||
"max_event_timestamp": "2026-01-01 00:10:00.000000",
|
||||
}
|
||||
}
|
||||
stats = {
|
||||
"events": "9",
|
||||
"visits": "4",
|
||||
"users": "3",
|
||||
"min_event_timestamp": "2026-01-01 00:00:00.000000",
|
||||
"max_event_timestamp": "2026-01-01 00:10:00.000000",
|
||||
}
|
||||
|
||||
assert compare_clickhouse_stats_to_manifest(manifest, stats) == ["events"]
|
||||
|
||||
|
||||
def test_raw_topic_value_can_keep_original_json_bytes(base_config):
|
||||
"""Raw value проверяется как JSON, но импортируется без пересборки строки."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
import_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
config = replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
)
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
artifact = builder.to_artifact(
|
||||
manifest=build_manifest(config=config, counters=builder.counters, state=state),
|
||||
state=state,
|
||||
)
|
||||
raw_value = (
|
||||
'{"event_timestamp":"2026-01-01 00:00:00.000000",'
|
||||
'"user_domain_id":"user-1","click_id":"click-1","event_id":"event-1"}'
|
||||
)
|
||||
artifact["raw_topics"]["browser_events"][0]["value_json"] = raw_value
|
||||
|
||||
class Publisher:
|
||||
def __init__(self):
|
||||
self.records = None
|
||||
|
||||
def publish_records(self, topic, records):
|
||||
if topic == "browser_events":
|
||||
self.records = records
|
||||
return len(records), 0
|
||||
|
||||
def flush(self):
|
||||
return None
|
||||
|
||||
class CompactWriter:
|
||||
def save(self, value):
|
||||
self.saved = value
|
||||
|
||||
def flush(self):
|
||||
return None
|
||||
|
||||
publisher = Publisher()
|
||||
import_startup_history_artifact(
|
||||
artifact,
|
||||
publisher=publisher,
|
||||
state_manager=CompactWriter(),
|
||||
manifest_manager=CompactWriter(),
|
||||
expected_config=config,
|
||||
)
|
||||
|
||||
assert publisher.records[0]["value_json"] == raw_value
|
||||
|
||||
|
||||
def test_artifact_v1_requires_raw_topics_for_import(base_config):
|
||||
"""Artifact v1.0 без raw value не может обещать byte-for-byte импорт."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
import_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
config = replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
)
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
artifact = builder.to_artifact(
|
||||
manifest=build_manifest(config=config, counters=builder.counters, state=state),
|
||||
state=state,
|
||||
)
|
||||
del artifact["raw_topics"]
|
||||
|
||||
class Publisher:
|
||||
def publish_records(self, topic, records):
|
||||
raise AssertionError("publish must not be called")
|
||||
|
||||
with pytest.raises(ValueError, match="raw_topics"):
|
||||
import_startup_history_artifact(
|
||||
artifact,
|
||||
publisher=Publisher(),
|
||||
state_manager=None,
|
||||
manifest_manager=None,
|
||||
expected_config=config,
|
||||
)
|
||||
|
||||
|
||||
def test_import_rejects_publisher_without_raw_publish_records(base_config):
|
||||
"""Импорт не должен пересобирать dict, если publisher не умеет raw records."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
import_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
config = replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
)
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
artifact = builder.to_artifact(
|
||||
manifest=build_manifest(config=config, counters=builder.counters, state=state),
|
||||
state=state,
|
||||
)
|
||||
artifact["raw_topics"]["browser_events"][0]["value_json"] = (
|
||||
'{"event_timestamp":"2026-01-01 00:00:00.000000",'
|
||||
'"user_domain_id":"user-1","click_id":"click-1","event_id":"event-1"}'
|
||||
)
|
||||
|
||||
class DictPublisher:
|
||||
def publish(self, topic, events):
|
||||
raise AssertionError("dict publish must not be called")
|
||||
|
||||
def flush(self):
|
||||
return None
|
||||
|
||||
with pytest.raises(TypeError, match="publish_records"):
|
||||
import_startup_history_artifact(
|
||||
artifact,
|
||||
publisher=DictPublisher(),
|
||||
state_manager=None,
|
||||
manifest_manager=None,
|
||||
expected_config=config,
|
||||
)
|
||||
|
||||
|
||||
def test_import_rejects_dirty_kafka_topics_before_publish(base_config):
|
||||
"""Повторный импорт не должен дописывать дубли в непустые data-топики."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
import_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
config = replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
)
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
artifact = builder.to_artifact(
|
||||
manifest=build_manifest(config=config, counters=builder.counters, state=state),
|
||||
state=state,
|
||||
)
|
||||
|
||||
class Publisher:
|
||||
def publish(self, topic, events):
|
||||
raise AssertionError("publish must not be called")
|
||||
|
||||
class DirtyTopics:
|
||||
def assert_data_topics_empty(self):
|
||||
raise RuntimeError("Kafka data topics are not empty")
|
||||
|
||||
with pytest.raises(RuntimeError, match="not empty"):
|
||||
import_startup_history_artifact(
|
||||
artifact,
|
||||
publisher=Publisher(),
|
||||
state_manager=None,
|
||||
manifest_manager=None,
|
||||
expected_config=config,
|
||||
topic_inspector=DirtyTopics(),
|
||||
)
|
||||
|
||||
|
||||
def test_import_rolls_back_kafka_topics_after_partial_publish(base_config):
|
||||
"""При частичной публикации импорт откатывает Kafka-топики до повтора."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
import_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
config = replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
)
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
artifact = builder.to_artifact(
|
||||
manifest=build_manifest(config=config, counters=builder.counters, state=state),
|
||||
state=state,
|
||||
)
|
||||
|
||||
class PartialPublisher:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def publish_records(self, topic, records):
|
||||
self.calls.append(topic)
|
||||
if topic == "location_events":
|
||||
raise RuntimeError("location down")
|
||||
return len(records), 0
|
||||
|
||||
class CompactWriter:
|
||||
def save(self, value):
|
||||
raise AssertionError("compact topics must not be written")
|
||||
|
||||
class Inspector:
|
||||
def __init__(self):
|
||||
self.snapshot = None
|
||||
self.rollback = None
|
||||
|
||||
def assert_data_topics_empty(self):
|
||||
return None
|
||||
|
||||
def snapshot_import_topics(self):
|
||||
self.snapshot = {"topics": ["browser_events", "location_events"]}
|
||||
return self.snapshot
|
||||
|
||||
def rollback_import_topics(self, snapshot):
|
||||
self.rollback = snapshot
|
||||
|
||||
publisher = PartialPublisher()
|
||||
inspector = Inspector()
|
||||
|
||||
with pytest.raises(RuntimeError, match="location down"):
|
||||
import_startup_history_artifact(
|
||||
artifact,
|
||||
publisher=publisher,
|
||||
state_manager=CompactWriter(),
|
||||
manifest_manager=CompactWriter(),
|
||||
expected_config=config,
|
||||
topic_inspector=inspector,
|
||||
)
|
||||
|
||||
assert publisher.calls == ["browser_events", "location_events"]
|
||||
assert inspector.rollback == inspector.snapshot
|
||||
|
||||
|
||||
def test_import_replays_events_and_compact_topics(base_config):
|
||||
"""Импорт пишет события в Kafka и служебные compact-топики, не трогая ClickHouse."""
|
||||
from clickstream_generator.startup_history_artifact import (
|
||||
StartupHistoryArtifactBuilder,
|
||||
build_manifest,
|
||||
import_startup_history_artifact,
|
||||
)
|
||||
|
||||
state = _state()
|
||||
config = replace(
|
||||
base_config,
|
||||
model_t0=state.model_t0,
|
||||
model_t_end=state.model_timestamp,
|
||||
model_time_speed=1,
|
||||
model_timezone="UTC",
|
||||
seed=42,
|
||||
)
|
||||
builder = StartupHistoryArtifactBuilder()
|
||||
builder.add_batch(_batch())
|
||||
artifact = builder.to_artifact(
|
||||
manifest=build_manifest(config=config, counters=builder.counters, state=state),
|
||||
state=state,
|
||||
)
|
||||
|
||||
class Publisher:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
self.flushed = False
|
||||
|
||||
def publish_records(self, topic, records):
|
||||
self.calls.append((topic, records))
|
||||
return len(records), 0
|
||||
|
||||
def flush(self):
|
||||
self.flushed = True
|
||||
|
||||
class CompactWriter:
|
||||
def __init__(self):
|
||||
self.saved = None
|
||||
self.flushed = False
|
||||
|
||||
def save(self, value):
|
||||
self.saved = value
|
||||
|
||||
def flush(self):
|
||||
self.flushed = True
|
||||
|
||||
publisher = Publisher()
|
||||
state_manager = CompactWriter()
|
||||
manifest_manager = CompactWriter()
|
||||
|
||||
result = import_startup_history_artifact(
|
||||
artifact,
|
||||
publisher=publisher,
|
||||
state_manager=state_manager,
|
||||
manifest_manager=manifest_manager,
|
||||
expected_config=config,
|
||||
)
|
||||
|
||||
assert [topic for topic, _ in publisher.calls] == [
|
||||
"browser_events",
|
||||
"location_events",
|
||||
"device_events",
|
||||
"geo_events",
|
||||
]
|
||||
assert publisher.calls[0][1][0]["value_json"].encode("utf-8") == (
|
||||
b'{"event_id": "event-1", "click_id": "click-1", '
|
||||
b'"user_domain_id": "user-1", '
|
||||
b'"event_timestamp": "2026-01-01 00:00:00.000000"}'
|
||||
)
|
||||
assert result["events"] == 4
|
||||
assert state_manager.saved.last_batch_id == state.last_batch_id
|
||||
assert manifest_manager.saved["state"]["last_batch_id"] == state.last_batch_id
|
||||
assert publisher.flushed
|
||||
assert state_manager.flushed
|
||||
assert manifest_manager.flushed
|
||||
Reference in New Issue
Block a user