fix(generator): устранён флаки runtime-гейта стыка backfill/live
- Зачем:
- гейт generated-history-runtime-check падал через раз: docker compose
stop убивал генератор SIGKILL'ом посреди batch (SIGTERM не ловился),
непарные строки маскировались под «смену фактуры» (задача 20).
- Что:
- генератор грациозно завершается по SIGTERM: текущий batch дописывается
во все топики с flush и записью history; compose даёт минуту grace.
- runtime-check ждёт пересекающий визит в STG (предусловие проверки),
seam-check получил precheck непарных live-строк; в STG-запросах
закреплён 'UTC' против сдвига наивных меток в поясе сервера.
- контрактные тесты усилены, задача 20 закрыта, блокер задачи 13 снят.
- Проверка:
- тесты: 192 passed (generator), 21 passed (контракты);
- стенд: 3 подряд зелёных make generated-history-runtime-check;
красный сценарий (искажение referer_url пересекающего визита) валит
гейт прежним сообщением при нулевых непарных счётчиках.
This commit is contained in:
@@ -335,6 +335,114 @@ if [[ "${should_check_live_seam}" == "1" ]]; then
|
||||
echo ""
|
||||
echo "=== Проверка ClickHouse: стык backfill/live в DDS ==="
|
||||
|
||||
live_pairing_query="
|
||||
WITH
|
||||
parseDateTime64BestEffort('${manifest_model_t_end}', 6, 'UTC') AS t_end,
|
||||
t_end + INTERVAL ${GEN_LIVE_CHECK_MINUTES} MINUTE AS t_live_end,
|
||||
live_browser AS (
|
||||
SELECT
|
||||
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
|
||||
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id
|
||||
FROM stg.browser_raw
|
||||
WHERE parseDateTime64BestEffortOrNull(
|
||||
JSONExtractString(raw, 'event_timestamp'), 6, 'UTC'
|
||||
) >= t_end
|
||||
AND parseDateTime64BestEffortOrNull(
|
||||
JSONExtractString(raw, 'event_timestamp'), 6, 'UTC'
|
||||
) < t_live_end
|
||||
AND event_id IS NOT NULL
|
||||
AND click_id IS NOT NULL
|
||||
GROUP BY event_id, click_id
|
||||
),
|
||||
live_click_ids AS (
|
||||
SELECT click_id
|
||||
FROM live_browser
|
||||
GROUP BY click_id
|
||||
),
|
||||
browser_counts AS (
|
||||
SELECT click_id, count() AS browser_rows
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
|
||||
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id
|
||||
FROM stg.browser_raw
|
||||
WHERE event_id IS NOT NULL AND click_id IS NOT NULL
|
||||
GROUP BY event_id, click_id
|
||||
)
|
||||
WHERE click_id IN (SELECT click_id FROM live_click_ids)
|
||||
GROUP BY click_id
|
||||
),
|
||||
location_ids AS (
|
||||
SELECT toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id
|
||||
FROM stg.location_raw
|
||||
WHERE event_id IS NOT NULL
|
||||
GROUP BY event_id
|
||||
),
|
||||
device_counts AS (
|
||||
SELECT
|
||||
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
||||
count() AS device_rows
|
||||
FROM stg.device_raw
|
||||
WHERE click_id IS NOT NULL
|
||||
AND click_id IN (SELECT click_id FROM live_click_ids)
|
||||
GROUP BY click_id
|
||||
),
|
||||
geo_counts AS (
|
||||
SELECT
|
||||
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
||||
count() AS geo_rows
|
||||
FROM stg.geo_raw
|
||||
WHERE click_id IS NOT NULL
|
||||
AND click_id IN (SELECT click_id FROM live_click_ids)
|
||||
GROUP BY click_id
|
||||
)
|
||||
SELECT
|
||||
(SELECT count() FROM live_browser) AS live_browser_rows,
|
||||
(
|
||||
SELECT count()
|
||||
FROM live_browser AS b
|
||||
LEFT JOIN location_ids AS l ON l.event_id = b.event_id
|
||||
WHERE l.event_id IS NULL
|
||||
) AS unpaired_location_rows,
|
||||
(
|
||||
SELECT ifNull(sum(if(
|
||||
b.browser_rows > ifNull(d.device_rows, 0),
|
||||
b.browser_rows - ifNull(d.device_rows, 0),
|
||||
0
|
||||
)), 0)
|
||||
FROM browser_counts AS b
|
||||
LEFT JOIN device_counts AS d ON d.click_id = b.click_id
|
||||
) AS unpaired_device_rows,
|
||||
(
|
||||
SELECT ifNull(sum(if(
|
||||
b.browser_rows > ifNull(g.geo_rows, 0),
|
||||
b.browser_rows - ifNull(g.geo_rows, 0),
|
||||
0
|
||||
)), 0)
|
||||
FROM browser_counts AS b
|
||||
LEFT JOIN geo_counts AS g ON g.click_id = b.click_id
|
||||
) AS unpaired_geo_rows
|
||||
SETTINGS join_use_nulls = 1
|
||||
FORMAT TabSeparated"
|
||||
|
||||
live_pairing="$(ch_query "${live_pairing_query}")"
|
||||
IFS=$'\t' read -r live_browser_rows unpaired_location_rows unpaired_device_rows unpaired_geo_rows <<< "${live_pairing}"
|
||||
|
||||
[[ "${live_browser_rows}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${unpaired_location_rows}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${unpaired_device_rows}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${unpaired_geo_rows}" =~ ^[0-9]+$ ]] \
|
||||
|| fail "не удалось прочитать пары live-строк"
|
||||
if (( unpaired_location_rows > 0 || unpaired_device_rows > 0 || unpaired_geo_rows > 0 )); then
|
||||
fail "непарные live-строки: location=${unpaired_location_rows}, device=${unpaired_device_rows}, geo=${unpaired_geo_rows} из browser=${live_browser_rows}"
|
||||
fi
|
||||
|
||||
echo "live_browser_rows=${live_browser_rows}"
|
||||
echo "unpaired_location_rows=${unpaired_location_rows}"
|
||||
echo "unpaired_device_rows=${unpaired_device_rows}"
|
||||
echo "unpaired_geo_rows=${unpaired_geo_rows}"
|
||||
|
||||
seam_duplicates_query="
|
||||
WITH
|
||||
toDateTime64('${CH_MODEL_T0}', 6) AS t0,
|
||||
|
||||
Reference in New Issue
Block a user