ddmitry/clickstream-ch-kafka-superset-demo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(generator): сохранена фактура визита при восстановлении

Browse Source 
- Зачем:
  - визит после восстановления не должен менять браузер и источники перехода внутри одного click_id.
- Что:
  - добавлен base_click_id в state v3 для восстановления донора фактуры.
  - исправлено восстановление timestamp offset без потери микросекунд.
  - расширены тесты и стыковая проверка browser/source и device/os/geo.
- Проверка:
  - uv run --with-requirements generator/requirements.txt pytest generator/tests -q.
  - bash -n scripts/check_generated_analytics.sh.
  - git diff --cached --check.
This commit is contained in:
ddadmin
2026-07-04 21:46:23 +03:00
parent 0cbfe9b32e
commit e2d06841de
13 changed files with 464 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/tests/test_state.py
+67 -38
View File
@@ -18,8 +18,8 @@ def _make_valid_rng_state(seed: int = 42):
return rng.getstate()
def _make_valid_v2_state_data() -> dict:
"""Создаёт минимальный валидный state v2 для тестов загрузки."""
def _make_valid_state_data() -> dict:
"""Создаёт минимальный валидный state для тестов загрузки."""
return {
"tick": 42,
"rng_state": list(_make_valid_rng_state(42)),
@@ -31,7 +31,7 @@ def _make_valid_v2_state_data() -> dict:
"model_timezone": "UTC",
"model_t0": "2026-01-01T00:00:00+00:00",
"gen_seed": 42,
"version": "2.0",
"version": "3.0",
"population": [
{
"user_domain_id": "user-1",
@@ -44,6 +44,7 @@ def _make_valid_v2_state_data() -> dict:
{
"user_domain_id": "user-1",
"click_id": "visit-1",
"base_click_id": "seed-1",
"next_index": 1,
"started_at": "2026-06-11T12:00:00",
"offsets_us": [0, 60_000_000],
@@ -66,7 +67,7 @@ def _minimal_population() -> list[dict]:
def _with_resume_fields(data: dict) -> dict:
"""Добавляет обязательные поля state v2, не связанные с проверяемой ошибкой."""
"""Добавляет обязательные поля state, не связанные с проверяемой ошибкой."""
return {
**data,
"model_timestamp": "2026-01-01T10:00:00+00:00",
@@ -97,7 +98,7 @@ class TestGeneratorState:
model_timezone="UTC",
model_t0=now,
gen_seed=42,
version="2.0",
version="3.0",
)
assert state.tick == 42
@@ -110,7 +111,7 @@ class TestGeneratorState:
assert state.model_timezone == "UTC"
assert state.model_t0 == now
assert state.gen_seed == 42
assert state.version == "2.0"
assert state.version == "3.0"
def test_default_version(self):
"""Новые состояния по умолчанию пишутся в версии 2."""
@@ -124,7 +125,7 @@ class TestGeneratorState:
last_timestamp=now,
)
assert state.version == "2.0"
assert state.version == "3.0"
def test_to_dict_serialization(self):
"""Сериализация в словарь (JSON-safe, без pickle)."""
@@ -150,7 +151,7 @@ class TestGeneratorState:
assert data["model_timezone"] == "UTC"
assert data["model_t0"] == now.isoformat()
assert data["gen_seed"] is None
assert data["version"] == "2.0"
assert data["version"] == "3.0"
# Проверяем что rng_state сериализован как tuple (JSON-safe, без pickle)
assert "rng_state" in data
@@ -225,8 +226,8 @@ class TestGeneratorState:
assert next_values == values_after
def test_version_2_roundtrip_keeps_population_and_active_visits(self):
"""State v2 хранит популяцию и активные визиты в JSON."""
def test_state_roundtrip_keeps_population_and_active_visits(self):
"""State хранит популяцию и активные визиты в JSON."""
rng_state = _make_valid_rng_state(42)
state = GeneratorState(
tick=7,
@@ -239,7 +240,7 @@ class TestGeneratorState:
model_timezone="Europe/Moscow",
model_t0=datetime(2026, 1, 1, 0, 0, tzinfo=timezone.utc),
gen_seed=42,
version="2.0",
version="3.0",
population=[
{
"user_domain_id": "user-1",
@@ -252,6 +253,7 @@ class TestGeneratorState:
{
"user_domain_id": "user-1",
"click_id": "visit-1",
"base_click_id": "seed-1",
"next_index": 1,
"started_at": "2026-06-11T12:00:00",
"offsets_us": [0, 60_000_000],
@@ -263,7 +265,7 @@ class TestGeneratorState:
restored = GeneratorState.from_dict(json.loads(json.dumps(state.to_dict())))
assert restored.version == "2.0"
assert restored.version == "3.0"
assert restored.tick == state.tick
assert restored.rng_state == rng_state
assert restored.population == state.population
@@ -281,7 +283,7 @@ class TestGeneratorStateValidation:
"""Тесты валидации состояния и graceful degradation."""
def test_from_dict_missing_version_raises(self):
"""from_dict выбрасывает исключение при отсутствии версии v2."""
"""from_dict выбрасывает исключение при отсутствии версии."""
data = {
"tick": 42,
"last_batch_id": "test",
@@ -298,7 +300,7 @@ class TestGeneratorStateValidation:
"rng_state": "not_a_tuple",
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -313,7 +315,7 @@ class TestGeneratorStateValidation:
"rng_state": [1], # Слишком короткий
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -328,7 +330,7 @@ class TestGeneratorStateValidation:
"rng_state": [999, [1, 2, 3], None], # Невалидный state
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -343,7 +345,7 @@ class TestGeneratorStateValidation:
"rng_state": "invalid",
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
})
@@ -365,7 +367,7 @@ class TestGeneratorStateValidation:
"model_timezone": "UTC",
"model_t0": "2026-01-01T00:00:00+00:00",
"gen_seed": 42,
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
}
@@ -378,7 +380,7 @@ class TestGeneratorStateValidation:
def test_from_dict_safe_returns_none_on_invalid_gen_seed(self):
"""gen_seed в JSON state должен быть числом или null."""
data = _make_valid_v2_state_data()
data = _make_valid_state_data()
data["gen_seed"] = "42"
result = GeneratorState.from_dict_safe(data)
@@ -387,7 +389,7 @@ class TestGeneratorStateValidation:
def test_from_dict_safe_returns_none_on_bool_model_time_speed(self):
"""model_time_speed не принимает bool как числовую скорость."""
data = _make_valid_v2_state_data()
data = _make_valid_state_data()
data["model_time_speed"] = True
result = GeneratorState.from_dict_safe(data)
@@ -395,14 +397,14 @@ class TestGeneratorStateValidation:
assert result is None
def test_from_dict_safe_returns_none_without_model_resume_fields(self):
"""State v2 без связки модельного и настенного времени несовместим."""
"""State без связки модельного и настенного времени несовместим."""
rng = random.Random(42)
data = {
"tick": 42,
"rng_state": list(rng.getstate()),
"last_batch_id": "test",
"last_timestamp": "2024-01-01T00:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": _minimal_population(),
"active_visits": [],
}
@@ -497,7 +499,7 @@ class TestKafkaStateManager:
rng_state=_make_valid_rng_state(100),
last_batch_id="xyz789",
last_timestamp=now,
version="2.0",
version="3.0",
population=_minimal_population(),
)
@@ -540,8 +542,8 @@ class TestKafkaStateManager:
# Должно вернуть None из-за невалидного state
assert result is None
def test_load_invalid_v2_nested_state_returns_none(self, caplog):
"""Битое state v2 с валидным rng_state даёт чистый старт."""
def test_load_invalid_nested_state_returns_none(self, caplog):
"""Битое state с валидным rng_state даёт чистый старт."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -555,12 +557,13 @@ class TestKafkaStateManager:
"rng_state": list(_make_valid_rng_state(42)),
"last_batch_id": "bad-v2",
"last_timestamp": "2026-06-11T12:00:00+00:00",
"version": "2.0",
"version": "3.0",
"population": [{"user_domain_id": "user-1"}],
"active_visits": [
{
"user_domain_id": "user-1",
"click_id": "visit-1",
"base_click_id": "seed-1",
"next_index": 1,
"started_at": "2026-06-11T12:00:00",
"offsets_us": [0],
@@ -579,15 +582,15 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_empty_population_v2_returns_none(self, caplog):
"""Пустая популяция в state v2 не восстанавливается."""
def test_load_empty_population_returns_none(self, caplog):
"""Пустая популяция в state не восстанавливается."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["population"] = []
bad_state["active_visits"] = []
@@ -606,15 +609,15 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_bad_pending_births_v2_returns_none(self, caplog):
"""Нечисловой pending_visit_births в state v2 не восстанавливается."""
def test_load_bad_pending_births_returns_none(self, caplog):
"""Нечисловой pending_visit_births в state не восстанавливается."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["pending_visit_births"] = "bad"
mock_message = MagicMock()
@@ -632,7 +635,33 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_active_visit_with_unknown_user_v2_returns_none(self, caplog):
def test_load_active_visit_without_base_click_id_returns_none(self, caplog):
"""Активный визит без донора фактуры не восстанавливается."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_state_data()
del bad_state["active_visits"][0]["base_click_id"]
mock_message = MagicMock()
mock_message.key = b"default"
mock_message.value = bad_state
mock_consumer = MagicMock()
mock_consumer.__iter__ = MagicMock(return_value=iter([mock_message]))
mock_consumer_class.return_value = mock_consumer
manager = KafkaStateManager("kafka:29092")
with caplog.at_level(logging.WARNING, logger="generator"):
result = manager.load()
assert result is None
assert "base_click_id" in caplog.text
def test_load_active_visit_with_unknown_user_returns_none(self, caplog):
"""Активный визит должен ссылаться на пользователя из популяции."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -640,7 +669,7 @@ class TestKafkaStateManager:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["active_visits"][0]["user_domain_id"] = "missing-user"
mock_message = MagicMock()
@@ -658,7 +687,7 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_active_visit_with_conflicting_click_id_v2_returns_none(self, caplog):
def test_load_active_visit_with_conflicting_click_id_returns_none(self, caplog):
"""active_click_id пользователя не должен противоречить визиту."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -666,7 +695,7 @@ class TestKafkaStateManager:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["population"][0]["active_click_id"] = "other-visit"
mock_message = MagicMock()
@@ -684,7 +713,7 @@ class TestKafkaStateManager:
assert result is None
assert "Invalid state" in caplog.text
def test_load_population_ghost_active_click_id_v2_returns_none(self, caplog):
def test_load_population_ghost_active_click_id_returns_none(self, caplog):
"""active_click_id пользователя должен иметь соответствующий активный визит."""
with patch("generator._import_kafka") as mock_import, \
patch("kafka.KafkaConsumer") as mock_consumer_class:
@@ -692,7 +721,7 @@ class TestKafkaStateManager:
mock_producer_class = MagicMock()
mock_import.return_value = (mock_producer_class, None)
bad_state = _make_valid_v2_state_data()
bad_state = _make_valid_state_data()
bad_state["population"][0]["active_click_id"] = "ghost"
bad_state["active_visits"] = []