ddmitry/clickstream-ch-kafka-superset-demo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(generator): сохранена фактура визита при восстановлении

Browse Source 
- Зачем:
  - визит после восстановления не должен менять браузер и источники перехода внутри одного click_id.
- Что:
  - добавлен base_click_id в state v3 для восстановления донора фактуры.
  - исправлено восстановление timestamp offset без потери микросекунд.
  - расширены тесты и стыковая проверка browser/source и device/os/geo.
- Проверка:
  - uv run --with-requirements generator/requirements.txt pytest generator/tests -q.
  - bash -n scripts/check_generated_analytics.sh.
  - git diff --cached --check.
This commit is contained in:
ddadmin
2026-07-04 21:46:23 +03:00
parent 0cbfe9b32e
commit e2d06841de
13 changed files with 464 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/tests/test_generation.py
+116 -1
View File
@@ -18,6 +18,7 @@ from generator import (
generate_tick_batch,
hour_factor,
)
from clickstream_generator.runtime import _timestamp_to_state_offset
ALLOWED_PAGE_PATHS = {
@@ -37,6 +38,24 @@ def _parse_event_timestamps(batch):
]
def _without_fields(batch, excluded_fields):
return {
topic: [
{
key: value
for key, value in event.items()
if key not in excluded_fields
}
for event in events
]
for topic, events in batch.items()
}
def _without_event_ids(batch):
return _without_fields(batch, {"event_id"})
def _page_path_visits(generator, visits_count: int):
return [
[
@@ -297,7 +316,7 @@ class TestEventGeneration:
def test_tick_stream_state_stays_compact_at_active_session_limit(
self, event_dictionary, base_config
):
"""State v2 не хранит полные события активных визитов."""
"""State не хранит полные события активных визитов."""
config = replace(
base_config,
max_session_events=30,
@@ -386,6 +405,102 @@ class TestEventGeneration:
for timestamp in resumed_timestamps
)
def test_tick_stream_restored_visit_keeps_per_event_fixture(
self, event_dictionary, base_config
):
"""Восстановленный визит продолжает ту же фактуру событий."""
config = replace(base_config, max_session_events=8)
tick_at = datetime(2026, 6, 11, 12, 0)
resume_at = tick_at + timedelta(hours=2)
generator = EventGenerator(event_dictionary, config)
stream = TickStreamGenerator(generator)
stream.generate_tick(event_budget=10, tick_started_at=tick_at)
state = stream.to_state(
tick=1,
rng_state=generator.rng.getstate(),
last_batch_id="batch-1",
last_timestamp=tick_at,
)
restored_state = type(state).from_dict(json.loads(json.dumps(state.to_dict())))
restored_generator = EventGenerator(event_dictionary, config)
restored_stream = TickStreamGenerator(restored_generator)
restored_stream.restore_state(restored_state, resume_model_at=tick_at)
uninterrupted = stream.generate_tick(event_budget=0, tick_started_at=resume_at)
restored = restored_stream.generate_tick(event_budget=0, tick_started_at=resume_at)
assert restored["browser_events"]
assert _without_event_ids(restored) == _without_event_ids(uninterrupted)
def test_tick_stream_restored_visit_keeps_cyclic_fallback_fixture(
self, event_dictionary, base_config
):
"""Короткий донор фактуры после восстановления повторяется так же."""
config = replace(base_config, max_session_events=30)
tick_at = datetime(2026, 6, 11, 12, 0)
resume_at = tick_at + timedelta(hours=12)
generator = EventGenerator(event_dictionary, config)
stream = TickStreamGenerator(generator)
stream.generate_tick(event_budget=100, tick_started_at=tick_at)
state = stream.to_state(
tick=1,
rng_state=generator.rng.getstate(),
last_batch_id="batch-1",
last_timestamp=tick_at,
)
assert any(
len(event_dictionary.browser_by_click_id[visit["base_click_id"]])
< len(visit["offsets_us"])
for visit in state.active_visits
)
restored_state = type(state).from_dict(json.loads(json.dumps(state.to_dict())))
restored_generator = EventGenerator(event_dictionary, config)
restored_stream = TickStreamGenerator(restored_generator)
restored_stream.restore_state(restored_state, resume_model_at=tick_at)
uninterrupted = stream.generate_tick(event_budget=0, tick_started_at=resume_at)
restored = restored_stream.generate_tick(event_budget=0, tick_started_at=resume_at)
assert restored["browser_events"]
assert _without_event_ids(restored) == _without_event_ids(uninterrupted)
def test_state_offset_preserves_microseconds_without_float_rounding(self):
"""Смещение state не теряет микросекунду на float-округлении."""
started_at = datetime(2026, 6, 11, 12, 0, 0)
timestamp = datetime(2026, 6, 11, 12, 8, 34, 130779)
assert _timestamp_to_state_offset(started_at, timestamp) == 514_130_779
def test_tick_stream_restore_rejects_unknown_fixture_donor(
self, event_dictionary, base_config
):
"""Неизвестный донор фактуры в state даёт ошибку восстановления."""
config = replace(base_config, max_session_events=5)
tick_at = datetime(2026, 6, 11, 12, 0)
generator = EventGenerator(event_dictionary, config)
stream = TickStreamGenerator(generator)
stream.generate_tick(event_budget=10, tick_started_at=tick_at)
state = stream.to_state(
tick=1,
rng_state=generator.rng.getstate(),
last_batch_id="batch-1",
last_timestamp=tick_at,
)
state_data = json.loads(json.dumps(state.to_dict()))
state_data["active_visits"][0]["base_click_id"] = "missing-click-id"
restored_state = type(state).from_dict(state_data)
restored_generator = EventGenerator(event_dictionary, config)
restored_stream = TickStreamGenerator(restored_generator)
with pytest.raises(ValueError, match="Unknown fixture base_click_id"):
restored_stream.restore_state(restored_state, resume_model_at=tick_at)
def test_tick_stream_restored_after_long_idle_closes_overdue_visit_without_replay(
self, event_dictionary, base_config
):
generator/tests/test_service.py
+10 -10
View File
@@ -926,11 +926,11 @@ class TestGeneratorServiceBackfill:
}
class TestGeneratorServiceStateV2:
"""Тесты подключения state v2 к сервисному запуску."""
class TestGeneratorServiceState:
"""Тесты подключения state к сервисному запуску."""
def test_start_restores_tick_stream_state_v2(self, base_config, event_dictionary):
"""Сервис восстанавливает популяцию и активные визиты из state v2."""
def test_start_restores_tick_stream_state(self, base_config, event_dictionary):
"""Сервис восстанавливает популяцию и активные визиты из state."""
source_generator = EventGenerator(event_dictionary, base_config)
source_stream = TickStreamGenerator(source_generator)
tick_at = datetime.now(timezone.utc).replace(tzinfo=None)
@@ -1073,8 +1073,8 @@ class TestGeneratorServiceStateV2:
assert service._model_time == model_t_end
assert service.stream.active_visit_count == source_stream.active_visit_count
def test_save_state_writes_tick_stream_state_v2(self, base_config):
"""Сервис сохраняет v2-снимок тикового слоя."""
def test_save_state_writes_tick_stream_state(self, base_config):
"""Сервис сохраняет снимок тикового слоя."""
service = GeneratorService(base_config)
service.state_manager = MagicMock()
tick_at = datetime.now(timezone.utc).replace(tzinfo=None)
@@ -1085,7 +1085,7 @@ class TestGeneratorServiceStateV2:
service._save_state("batch-1")
saved_state = service.state_manager.save.call_args.args[0]
assert saved_state.version == "2.0"
assert saved_state.version == "3.0"
assert saved_state.model_timestamp == tick_at.replace(tzinfo=timezone.utc)
assert saved_state.wall_timestamp.tzinfo is not None
assert saved_state.model_time_speed == base_config.model_time_speed
@@ -1162,13 +1162,13 @@ class TestGeneratorServiceStateV2:
state_manager.load.assert_not_called()
assert service._tick == 0
def test_invalid_restored_v2_state_starts_fresh(self, base_config, caplog):
"""Сервис не падает, если v2 state ссылается на неизвестный профиль."""
def test_invalid_restored_state_starts_fresh(self, base_config, caplog):
"""Сервис не падает, если state ссылается на неизвестный профиль."""
state_manager = MagicMock()
state_manager.load.return_value = GeneratorState(
tick=9,
rng_state=random.Random(42).getstate(),
last_batch_id="bad-v2",
last_batch_id="bad-state",
last_timestamp=datetime.now(timezone.utc),
model_timestamp=base_config.model_t0,
wall_timestamp=datetime.now(timezone.utc),
generator/tests/test_state.py
+67 -38
View File
@@ -18,8 +18,8 @@ def _make_valid_rng_state(seed: int = 42):
return rng.getstate()
def _make_valid_v2_state_data() -> dict:
"""Создаёт минимальный валидный state v2 для тестов загрузки."""
def _make_valid_state_data() -> dict:
"""Создаёт минимальный валидный state для тестов загрузки."""
return {
"tick": 42,
"rng_state": list(_make_valid_rng_state(42)),
@@ -31,7 +31,7 @@ def _make_valid_v2_state_data() -> dict:
"model_timezone": "UTC",
"model_t0": "2026-01-01T00:00:00+00:00",
"gen_seed": 42,
"version": "2.0",
"version": "3.0",
"population": [
{
"user_domain_id": "user-1",
@@ -44,6 +44,7 @@ def _make_valid_v2_state_data() -> dict:
{
"user_domain_id": "user-1",
"click_id": "visit-1",
"base_click_id": "seed-1",
"next_index": 1,
"started_at": "2026-06-11T12:00:00",
"offsets_us": [0, 60_000_000],
@@ -66,7 +67,7 @@ def _minimal_population() -> list[dict]:
def _with_resume_fields(data: dict) -> dict:
"""Добавляет обязательные поля state v2, не связанные с проверяемой ошибкой."""
"""Добавляет обязательные поля state, не связанные с проверяемой ошибкой."""
return {
**data,
"model_timestamp": "2026-01-01T10:00:00+00:00",
@@ -97,7 +98,7 @@ class TestGeneratorState:
model_timezone="UTC",
model_t0=now,
gen_seed=42,
version="2.0",
version="3.0",
)
assert state.tick == 42
@@ -110,7 +111,7 @@ class TestGeneratorState:
assert state.model_timezone == "UTC"
assert state.model_t0 == now
assert state.gen_seed == 42
assert state.version == "2.0"
assert state.version == "3.0"
def test_default_version(self):
"""Новые состояния по умолчанию пишутся в версии 2."""
@@ -124,7 +125,7 @@ class TestGeneratorState:
last_timestamp=now,
)
assert state.version == "2.0"
assert state.version == "3.0"
def test_to_dict_serialization(self):
"""Сериализация в словарь (JSON-safe, без pickle)."""
@@ -150,7 +151,7 @@ class TestGeneratorState:
assert data["model_timezone"] == "UTC"
assert data["model_t0"] == now.isoformat()
assert data["gen_seed"] is None
assert data["version"] == "2.0"
assert data["version"] == "3.0"
# Проверяем что rng_state сериализован как tuple (JSON-safe, без pickle)
assert "rng_state" in data
@@ -225,8 +226,8 @@ class TestGeneratorState:
assert next_values == values_after
def test_version_2_roundtrip_keeps_population_and_active_visits(self):
"""State v2 хранит популяцию и активные визиты в JSON."""
def test_state_roundtrip_keeps_population_and_active_visits(self):
"""State хранит популяцию и активные визиты в JSON."""
rng_state = _make_valid_rng_state(42)
state = GeneratorState(
tick=7,
@@ -239,7 +240,7 @@ class TestGeneratorState:
model_timezone="Europe/Moscow",
model_t0=datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc),
gen_seed=42,
version="2.0",
version="3.0",
population=[
{
"user_domain_id": "user-1",
@@ -252,6 +253,7 @@ class TestGeneratorState:
{
"user_domain_id": "user-1",
"click_id": "visit-1",
"base_click_id": "seed-1",
"next_index": 1,
"started_at": "2026-06-11T12:00:00",
"offsets_us": [0, 60_000_000],
@@ -263,7 +265,7 @@ class TestGeneratorState:
restored = GeneratorState.from_dict(json.loads(json.dumps(state.to_dict())))
assert restored.version == "2.0"
assert restored.version == "3.0"
assert restored.tick == state.tick
assert restored.rng_state == rng_state
assert restored.population == state.population
@@ -281,7 +283,7 @@ class TestGeneratorStateValidation:
"""Тесты валидации состояния и graceful degradation."""
def test_from_dict_missing_version_raises(self):
"""from_dict выбрасывает исключение при отсутствии версии v2."""
"""from_dict выбрасывает исключение при отсутствии версии."""
data = {
"tick": 42,
"last_batch_id": "test",
@@ -298,7 +300,7 @@ class TestGeneratorStateValidation:
"rng_state": "not_a_tuple",
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -313,7 +315,7 @@ class TestGeneratorStateValidation:
"rng_state": [1], # Слишком короткий
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -328,7 +330,7 @@ class TestGeneratorStateValidation:
"rng_state": [999, [1, 2, 3], None], # Невалидный state
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -343,7 +345,7 @@ class TestGeneratorStateValidation:
"rng_state": "invalid",
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -365,7 +367,7 @@ class TestGeneratorStateValidation:
"model_timezone": "UTC",
"model_t0": "2026-01-01T00:00:00+00:00",
"gen_seed": 42,
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
}
@@ -378,7 +380,7 @@ class TestGeneratorStateValidation:
def test_from_dict_safe_returns_none_on_invalid_gen_seed(self):
"""gen_seed в JSON state должен быть числом или null."""
data = _make_valid_v2_state_data()
data = _make_valid_state_data()
data["gen_seed"] = "42"
result = GeneratorState.from_dict_safe(data)
@@ -387,7 +389,7 @@ class TestGeneratorStateValidation:
def test_from_dict_safe_returns_none_on_bool_model_time_speed(self):
"""model_time_speed не принимает bool как числовую скорость."""
data = _make_valid_v2_state_data()
data = _make_valid_state_data()
data["model_time_speed"] = True
result = GeneratorState.from_dict_safe(data)
@@ -395,14 +397,14 @@ class TestGeneratorStateValidation:
assert result is None
def test_from_dict_safe_returns_none_without_model_resume_fields(self):
"""State v2 без связки модельного и настенного времени несовместим."""
"""State без связки модельного и настенного времени несовместим."""
rng = random.Random(42)
data = {
"tick": 42,
"rng_state": list(rng.getstate()),
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
}
@@ -497,7 +499,7 @@ class TestKafkaStateManager:
rng_state=_make_valid_rng_state(100),
last_batch_id="xyz789",
last_timestamp=now,
version="2.0",
version="3.0",
population=_minimal_population(),
)
@@ -540,8 +542,8 @@ class TestKafkaStateManager:
# Должно вернуть None из-за невалидного state
assert result is None
def test_load_invalid_v2_nested_state_returns_none(self, caplog):
"""Битое state v2 с валидным rng_state даёт чистый старт."""
def test_load_invalid_nested_state_returns_none(self, caplog):
"""Битое state с валидным rng_state даёт чистый старт."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -555,12 +557,13 @@ class TestKafkaStateManager:
"rng_state": list(_make_valid_rng_state(42)),
"last_batch_id": "bad-v2",
"last_timestamp": "2026-06-11T12:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": [{"user_domain_id": "user-1"}],
"active_visits": [
{
"user_domain_id": "user-1",
"click_id": "visit-1",
"base_click_id": "seed-1",
"next_index": 1,
"started_at": "2026-06-11T12:00:00",
"offsets_us": [0],
@@ -579,15 +582,15 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_empty_population_v2_returns_none(self, caplog):
"""Пустая популяция в state v2 не восстанавливается."""
def test_load_empty_population_returns_none(self, caplog):
"""Пустая популяция в state не восстанавливается."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["population"] = []
bad_state["active_visits"] = []
@@ -606,15 +609,15 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_bad_pending_births_v2_returns_none(self, caplog):
"""Нечисловой pending_visit_births в state v2 не восстанавливается."""
def test_load_bad_pending_births_returns_none(self, caplog):
"""Нечисловой pending_visit_births в state не восстанавливается."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["pending_visit_births"] = "bad"
mock_message = MagicMock()
@@ -632,7 +635,33 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_active_visit_with_unknown_user_v2_returns_none(self, caplog):
def test_load_active_visit_without_base_click_id_returns_none(self, caplog):
"""Активный визит без донора фактуры не восстанавливается."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_state_data()
del bad_state["active_visits"][0]["base_click_id"]
mock_message = MagicMock()
mock_message.key = b"default"
mock_message.value = bad_state
mock_consumer = MagicMock()
mock_consumer.__iter__ = MagicMock(return_value=iter([mock_message]))
mock_consumer_class.return_value = mock_consumer
manager = KafkaStateManager("kafka:29092")
with caplog.at_level(logging.WARNING, logger="generator"):
result = manager.load()
assert result is None
assert "base_click_id" in caplog.text
def test_load_active_visit_with_unknown_user_returns_none(self, caplog):
"""Активный визит должен ссылаться на пользователя из популяции."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -640,7 +669,7 @@ class TestKafkaStateManager:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["active_visits"][0]["user_domain_id"] = "missing-user"
mock_message = MagicMock()
@@ -658,7 +687,7 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_active_visit_with_conflicting_click_id_v2_returns_none(self, caplog):
def test_load_active_visit_with_conflicting_click_id_returns_none(self, caplog):
"""active_click_id пользователя не должен противоречить визиту."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -666,7 +695,7 @@ class TestKafkaStateManager:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["population"][0]["active_click_id"] = "other-visit"
mock_message = MagicMock()
@@ -684,7 +713,7 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_population_ghost_active_click_id_v2_returns_none(self, caplog):
def test_load_population_ghost_active_click_id_returns_none(self, caplog):
"""active_click_id пользователя должен иметь соответствующий активный визит."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -692,7 +721,7 @@ class TestKafkaStateManager:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["population"][0]["active_click_id"] = "ghost"
bad_state["active_visits"] = []