feat(generator): потеря и дубль purchase (B, D) и счетчики описи

- Зачем:
  - трекер стенда был полным, и правило «деньги — по бэкенду» держалось
    на слове; классы B и D делают недосчет и пересчет трекера
    наблюдаемыми (этап 6, #184).
- Что:
  - подпоток BEACONS: точечная потеря purchase (~3%) и сюжетный дубль
    (~2%) швом после _purchases, с гарантией моста идентичностей и
    условием допуска дубля по паузе до следующего визита;
  - опись считает пять классов по приоритету спеки; фильтр
    classes_vs_inventory расширен, нулевой класс читается нулем;
  - fate.md, seeds.md, dm.md и orders/README.md приведены к имени
    BEACONS и строгому правилу допуска.
- Проверка:
  - make lint, make typecheck, make test (433 passed) и make inventory
    из generator/ — хеши слепков не изменились;
  - на стенде: make smoke, make check-clickhouse, make check-services
    зеленые; в dq_summary classes_vs_inventory и sessions_vs_reference
    pass, витрина сверки показывает lost_event и duplicate_event.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

generator/tests/test_commerce.py

@@ -133,15 +133,15 @@ def opened_cards(events: day.Day) -> set[tuple[int, int]]:
     )
 
 
-def test_a_confirmation_in_the_stream_always_has_its_purchase(weekday: day.Day):
-    """Клиентская сторона честная: подтверждение без покупки — это потеря.
-
-    Потери и дубли стенд заводит намеренно и позже (этапы 4 и 6); здесь их
-    быть не должно ни одной. Обратное тоже верно: покупка без подтверждения
-    в потоке — заказ ниоткуда.
-    """
+def test_only_a_lost_purchase_leaves_a_confirmation_without_purchase(
+    weekday: day.Day,
+):
+    """Потеря выбрасывает purchase, но оставляет просмотр confirmation."""
     bought = set(rows_of(weekday, commerce.PURCHASE)["VisitID"].tolist())
-    assert confirmed_visits(weekday) == bought
+    missing = confirmed_visits(weekday) - bought
+    lost = int((weekday.purchase_outcome == commerce.EventOutcome.LOST).sum())
+    assert len(missing) == lost > 0
+    assert bought <= confirmed_visits(weekday)
 
 
 def test_the_day_boundary_takes_the_confirmation_together_with_its_purchase(
@@ -159,7 +159,10 @@ def test_the_day_boundary_takes_the_confirmation_together_with_its_purchase(
     events = day.stream(CANONICAL_SEED, WEEKDAY)
     bought = set(rows_of(events, commerce.PURCHASE)["VisitID"].tolist())
     assert bought
-    assert confirmed_visits(events) == bought
+    missing = confirmed_visits(events) - bought
+    lost = int((events.purchase_outcome == commerce.EventOutcome.LOST).sum())
+    assert len(missing) == lost
+    assert bought <= confirmed_visits(events)
 
 
 def test_a_purchase_carries_exactly_one_order_number(weekday: day.Day):
@@ -525,13 +528,15 @@ def test_the_goals_repeat_the_trade_events(weekday: day.Day):
 
 def test_the_order_number_reads_as_the_day_and_the_count(weekday: day.Day):
     """Номер — день и порядковый номер покупки в нём; он же `order_id` бэкенда."""
-    orders = rows_of(weekday, commerce.PURCHASE)
-    numbers = [cell[0] for cell in orders["purchaseID"]]
+    events = rows_of(weekday, commerce.PURCHASE)
+    numbers = [cell[0] for cell in events["purchaseID"]]
     date = str(weekday.columns["EventDate"][0]).replace("-", "")
-    assert numbers == [f"{date}-{place:04d}" for place in range(1, len(numbers) + 1)]
-    assert len(set(numbers)) == len(numbers)
-    # Нумерация идёт в порядке событий: строки дня уже упорядочены по времени.
-    assert list(orders["UTCEventTime"]) == sorted(orders["UTCEventTime"])
+    expected = [f"{date}-{place:04d}" for place in range(1, len(weekday.orders) + 1)]
+    # Порча идет после нумерации: потеря оставляет дыру, а дубль
+    # повторяет готовый номер и не двигает соседей.
+    assert list(weekday.orders.order_id) == expected
+    assert set(numbers) <= set(expected)
+    assert list(events["UTCEventTime"]) == sorted(events["UTCEventTime"])
 
 
 def test_the_coupon_comes_from_the_world_table_and_does_not_touch_the_revenue(
@@ -599,7 +604,23 @@ def test_the_trade_flow_draws_from_its_own_named_branch(
 
     monkeypatch.setattr(commerce, "day_stream", spy)
     day.stream(CANONICAL_SEED, WEEKDAY)
-    assert asked == [(CANONICAL_SEED, WEEKDAY, Component.COMMERCE)]
+    assert asked == [
+        (CANONICAL_SEED, WEEKDAY, Component.COMMERCE),
+        (CANONICAL_SEED, WEEKDAY, Component.BEACONS),
+    ]
+
+
+def test_an_assigned_purchase_is_never_lost(monkeypatch: pytest.MonkeyPatch):
+    """Гарантия плана сильнее порчи: мост склейки не теряется."""
+    monkeypatch.setattr(world, "LOST_PURCHASE_PERCENT", 100)
+    monkeypatch.setattr(world, "DUPLICATE_PURCHASE_PERCENT", 0)
+    events = day.stream(CANONICAL_SEED, WEEKDAY)
+    audience = plan.audience(CANONICAL_SEED, WEEKDAY)
+
+    promised = set(audience.client_id[audience.assigned_order].tolist())
+    bought = set(rows_of(events, commerce.PURCHASE)["ClientID"].tolist())
+    assert promised
+    assert bought == promised
 
 
 def test_the_promised_orders_of_a_pair_become_purchases(week: list[day.Day]):
@@ -627,6 +648,78 @@ def test_the_promised_orders_of_a_pair_become_purchases(week: list[day.Day]):
     assert pairs == plan.counters(CANONICAL_SEED, WEEK).pairs
 
 
+def test_a_duplicate_repeats_the_confirmation_story(weekday: day.Day):
+    """Обновление повторяет просмотр и purchase в том же визите."""
+    purchases = rows_of(weekday, commerce.PURCHASE)
+    ids_by_order: dict[str, list[int]] = {}
+    for row, cell in enumerate(purchases["purchaseID"]):
+        ids_by_order.setdefault(str(cell[0]), []).append(row)
+    duplicated = [rows for rows in ids_by_order.values() if len(rows) == 2]
+
+    expected = int((weekday.purchase_outcome == commerce.EventOutcome.DUPLICATED).sum())
+    assert len(duplicated) == expected > 0
+    for pair in duplicated:
+        first, second = sorted(pair, key=lambda row: purchases["UTCEventTime"][row])
+        delay = (
+            (purchases["UTCEventTime"][second] - purchases["UTCEventTime"][first])
+            .astype("timedelta64[s]")
+            .astype(int)
+        )
+        assert world.DUPLICATE_DELAY_SECONDS[0] <= delay
+        assert delay < world.DUPLICATE_DELAY_SECONDS[1]
+        assert delay < world.VISIT_TIMEOUT_SECONDS
+        assert purchases["VisitID"][first] == purchases["VisitID"][second]
+        assert purchases["WatchID"][first] != purchases["WatchID"][second]
+        for name in (
+            "purchaseID",
+            "purchaseRevenue",
+            "productID",
+            "productQuantity",
+            "ecommerce",
+        ):
+            left, right = purchases[name][first], purchases[name][second]
+            if isinstance(left, np.ndarray):
+                assert np.array_equal(left, right), name
+            else:
+                assert left == right, name
+
+        visit = purchases["VisitID"][first]
+        confirmation = (
+            (weekday.columns["EventType"] == "pageview")
+            & (weekday.page == Page.CONFIRMATION)
+            & (weekday.columns["VisitID"] == visit)
+        )
+        views = np.flatnonzero(confirmation)
+        assert views.size == 2
+        assert len(set(weekday.columns["WatchID"][views].tolist())) == 2
+        view_delay = (
+            (
+                weekday.columns["UTCEventTime"][views].max()
+                - weekday.columns["UTCEventTime"][views].min()
+            )
+            .astype("timedelta64[s]")
+            .astype(int)
+        )
+        assert view_delay == delay
+
+        cookie = purchases["ClientID"][first]
+        later_visit = (
+            (weekday.columns["ClientID"] == cookie)
+            & (weekday.columns["VisitID"] != visit)
+            & (weekday.columns["UTCEventTime"] > purchases["UTCEventTime"][second])
+        )
+        if np.any(later_visit):
+            gap = (
+                (
+                    weekday.columns["UTCEventTime"][later_visit].min()
+                    - purchases["UTCEventTime"][second]
+                )
+                .astype("timedelta64[s]")
+                .astype(int)
+            )
+            assert gap > world.VISIT_TIMEOUT_SECONDS
+
+
 def paired_cookies() -> set[int]:
     """Куки двухкуковых пар: их заказы обещаны планом, а не решены днём."""
     cookies: set[int] = set()
@@ -642,21 +735,26 @@ def test_flagged_buyers_buy_again_much_more_often(week: list[day.Day]):
     «Постоянный покупатель» читается только так — как кука, которая ходит
     неделями и покупает не раз: метки в событии нет и не будет.
 
-    Куки двухкуковых пар из счёта исключены: их заказы назначил план, и
-    вместе с ними лифт мерил бы гарантию, а не два новых рычага. Остаются
-    те помеченные, чьи покупки решил день: повторно покупают 4,6% из них
-    против 1,5% прочих — втрое чаще при пороге «вдвое».
+    Куки двухкуковых пар из счета исключены: их заказы назначил план, и
+    вместе с ними лифт мерил бы гарантию, а не два новых рычага. Даже после
+    событийной порчи оставшиеся помеченные покупают повторно более чем вдвое
+    чаще прочих.
     """
-    purchases: dict[int, int] = {}
+    purchases: dict[int, set[str]] = {}
     flagged: set[int] = set()
     for number, events in enumerate(week):
         audience = plan.audience(CANONICAL_SEED, number)
         flagged |= set(audience.client_id[audience.buyer].tolist())
-        for cookie in rows_of(events, commerce.PURCHASE)["ClientID"].tolist():
-            purchases[cookie] = purchases.get(cookie, 0) + 1
+        rows = rows_of(events, commerce.PURCHASE)
+        for cookie, order_id in zip(
+            rows["ClientID"].tolist(),
+            (str(cell[0]) for cell in rows["purchaseID"]),
+            strict=True,
+        ):
+            purchases.setdefault(cookie, set()).add(order_id)
 
     def repeat_share(cookies: set[int]) -> float:
-        again = sum(1 for cookie in cookies if purchases[cookie] > 1)
+        again = sum(1 for cookie in cookies if len(purchases[cookie]) > 1)
         return again / len(cookies)
 
     buyers = set(purchases)

generator/tests/test_day.py

@@ -88,6 +88,7 @@ def snapshot(events: day.Day) -> dict[str, Any]:
         "page": events.page.tolist(),
         "product": events.product.tolist(),
         "orders": orders(events),
+        "purchase_outcome": events.purchase_outcome.tolist(),
     }
 
 
@@ -115,6 +116,7 @@ def test_the_snapshot_notices_everything_the_day_hands_out(weekday: day.Day):
         page=weekday.page,
         product=weekday.product,
         orders=weekday.orders,
+        purchase_outcome=weekday.purchase_outcome,
     )
     assert snapshot(reordered) != original
 

generator/tests/test_inventory.py

@@ -48,7 +48,15 @@ def test_order_class_counters_cover_only_closed_days(inventory: dict):
     rows = [row for row in inventory["days"] if "orders" in row]
 
     assert [row["day"] for row in rows] == [0]
-    assert set(rows[0]["orders"]) == {"match", "cancelled", "amount_delta"}
+    assert set(rows[0]["orders"]) == {
+        "match",
+        "cancelled",
+        "lost_event",
+        "duplicate_event",
+        "amount_delta",
+    }
+    assert rows[0]["orders"]["lost_event"] > 0
+    assert rows[0]["orders"]["duplicate_event"] > 0
     today = day_module.stream(CANONICAL_SEED, rows[0]["day"])
     assert sum(rows[0]["orders"].values()) == len(today.orders)
 

generator/tests/test_orders.py

@@ -58,17 +58,33 @@ def user_id_by_order(events: day.Day) -> dict[str, int]:
     )
 
 
-def test_every_order_of_the_day_is_a_purchase_of_the_day(weekday: day.Day):
-    """Заказ — вторая проекция покупки, а не второе порождение.
+def purchase_row_by_order(
+    events: day.Day,
+) -> tuple[dict[str, NDArray[Any]], dict[str, int]]:
+    """Первая строка purchase каждого доехавшего заказа."""
+    rows = purchases_of(events)
+    index: dict[str, int] = {}
+    for row, cell in enumerate(rows["purchaseID"]):
+        index.setdefault(str(cell[0]), row)
+    return rows, index
 
-    У всякого заказа ровно одно событие `purchase`, и номер у них один:
-    сойтись двум источникам больше негде — сверка стенда соединяется
-    именно по нему.
-    """
+
+def test_every_order_of_the_day_is_a_purchase_of_the_day(weekday: day.Day):
+    """Заказ рождается до потери или дубля его событийной проекции."""
     events = purchases_of(weekday)
     numbers = [cell[0] for cell in events["purchaseID"]]
     assert len(numbers) > 100
-    assert list(weekday.orders.order_id) == numbers
+    lost = {
+        order_id
+        for order_id, outcome in zip(
+            weekday.orders.order_id, weekday.purchase_outcome, strict=True
+        )
+        if outcome == commerce.EventOutcome.LOST
+    }
+    assert set(numbers) == set(weekday.orders.order_id) - lost
+    assert len(numbers) - len(set(numbers)) == int(
+        (weekday.purchase_outcome == commerce.EventOutcome.DUPLICATED).sum()
+    )
 
 
 def dropped_line(client: list[tuple[Any, int]], order: list[tuple[Any, int]]):
@@ -88,16 +104,18 @@ def test_the_order_repeats_the_purchase_up_to_one_dropped_line(weekday: day.Day)
     """
     goods = catalog.catalog()
     price = dict(zip(goods.sku.tolist(), goods.price.tolist(), strict=True))
-    events = purchases_of(weekday)
+    events, row_by_order = purchase_row_by_order(weekday)
     money = weekday.orders
     deltas = 0
 
     for number, order in enumerate(weekday.orders.order_id):
-        assert order == events["purchaseID"][number][0]
+        if weekday.purchase_outcome[number] == commerce.EventOutcome.LOST:
+            continue
+        row = row_by_order[order]
         client = list(
             zip(
-                events["productID"][number].tolist(),
-                events["productQuantity"][number].tolist(),
+                events["productID"][row].tolist(),
+                events["productQuantity"][row].tolist(),
                 strict=True,
             )
         )
@@ -110,7 +128,7 @@ def test_the_order_repeats_the_purchase_up_to_one_dropped_line(weekday: day.Day)
         )
         # Выручка клиента и `items_total` бэкенда — одно число: в событии
         # оно дробное, у заказа целое в копейках.
-        revenue = round(events["purchaseRevenue"][number][0] * commerce.KOPECKS)
+        revenue = round(events["purchaseRevenue"][row][0] * commerce.KOPECKS)
         if mine == client:
             assert money.items_total[number] == revenue
             continue
@@ -136,15 +154,18 @@ def test_the_discount_comes_from_the_coupon_of_the_event(
 
     for today in start_world:
         money = today.orders
-        events = purchases_of(today)
-        codes = [cell[0] for cell in events["purchaseCoupon"]]
-        for number, code in enumerate(codes):
+        events, row_by_order = purchase_row_by_order(today)
+        for number, order_id in enumerate(money.order_id):
+            if today.purchase_outcome[number] == commerce.EventOutcome.LOST:
+                continue
+            row = row_by_order[order_id]
+            code = events["purchaseCoupon"][row][0]
             items_total = money.items_total[number]
             expected = items_total * percent[code] // 100 if code else 0
             assert money.discount[number] == expected, money.order_id[number]
             assert money.discount[number] < items_total
 
-            revenue = round(events["purchaseRevenue"][number][0] * commerce.KOPECKS)
+            revenue = round(events["purchaseRevenue"][row][0] * commerce.KOPECKS)
             discounted_delta += int(bool(code) and items_total != revenue)
 
     # Иначе проверка не отличила бы новую базу скидки от прежней.
@@ -255,8 +276,13 @@ def test_the_user_id_is_the_person_behind_the_cookie(weekday: day.Day):
         zip(audience.client_id.tolist(), audience.person_id.tolist(), strict=True)
     )
     events = purchases_of(weekday)
-    for number, cookie in enumerate(events["ClientID"].tolist()):
-        assert weekday.orders.user_id[number] == person[cookie]
+    by_order = user_id_by_order(weekday)
+    for order_id, cookie in zip(
+        (cell[0] for cell in events["purchaseID"]),
+        events["ClientID"].tolist(),
+        strict=True,
+    ):
+        assert by_order[order_id] == person[cookie]
     assert np.all(weekday.orders.user_id > 0)
     assert weekday.orders.user_id.max() < 2**53
 

generator/tests/test_seeds.py

@@ -78,7 +78,7 @@ def test_day_components_do_not_share_randomness():
         Component.TRAFFIC,
         Component.COMMERCE,
         Component.ORDERS,
-        Component.LATECOMERS,
+        Component.BEACONS,
     )
     assert len({int(component) for component in components}) == 4
     draws = {

generator/tests/test_snapshot.py

@@ -128,7 +128,9 @@ def test_the_record_matches_the_wire_contract(days: list[day_module.Day]):
         )
 
         if _born(record) == _date(SHORT_WINDOW_DAY):
-            assert record["created_at"][:19] + "Z" == bought[record["order_id"]]
+            # Потерянного purchase в трекере нет, но заказ остается.
+            if record["order_id"] in bought:
+                assert record["created_at"][:19] + "Z" == bought[record["order_id"]]
 
 
 def test_negative_money_does_not_leave_the_source(days: list[day_module.Day]):
@@ -325,7 +327,10 @@ def _purchase_times(today: day_module.Day) -> dict[str, str]:
     times = np.datetime_as_string(
         today.columns["UTCEventTime"][here], unit="s", timezone="UTC"
     )
-    return dict(zip(numbers, times.tolist(), strict=True))
+    result: dict[str, str] = {}
+    for number, moment in zip(numbers, times.tolist(), strict=True):
+        result.setdefault(number, moment)
+    return result
 
 
 def _born(record: dict) -> date:

generator/tests/test_world.py

@@ -122,6 +122,11 @@ def test_the_trade_event_never_overtakes_the_next_page():
     assert max(world.TRADE_DELAY_SECONDS) <= min(world.PAGE_PAUSE_SECONDS)
 
 
+def test_the_duplicate_delay_stays_inside_a_visit_timeout():
+    assert 0 < min(world.DUPLICATE_DELAY_SECONDS)
+    assert max(world.DUPLICATE_DELAY_SECONDS) <= world.VISIT_TIMEOUT_SECONDS
+
+
 def conversion(cart: int, checkout_of_cart: int) -> float:
     """Конверсия визита в заказ, проценты: произведение трёх шагов воронки."""
     return cart * checkout_of_cart * world.CONFIRMATION_OF_CHECKOUT_PERCENT / 100**2