fix(generator): устранён флаки runtime-гейта стыка backfill/live

- Зачем:
  - гейт generated-history-runtime-check падал через раз: docker compose
    stop убивал генератор SIGKILL'ом посреди batch (SIGTERM не ловился),
    непарные строки маскировались под «смену фактуры» (задача 20).
- Что:
  - генератор грациозно завершается по SIGTERM: текущий batch дописывается
    во все топики с flush и записью history; compose даёт минуту grace.
  - runtime-check ждёт пересекающий визит в STG (предусловие проверки),
    seam-check получил precheck непарных live-строк; в STG-запросах
    закреплён 'UTC' против сдвига наивных меток в поясе сервера.
  - контрактные тесты усилены, задача 20 закрыта, блокер задачи 13 снят.
- Проверка:
  - тесты: 192 passed (generator), 21 passed (контракты);
  - стенд: 3 подряд зелёных make generated-history-runtime-check;
    красный сценарий (искажение referer_url пересекающего визита) валит
    гейт прежним сообщением при нулевых непарных счётчиках.
This commit is contained in:
2026-07-12 19:33:28 +03:00
parent 55a3feb63d
commit dab120a07e
9 changed files with 433 additions and 30 deletions
+108
View File
@@ -335,6 +335,114 @@ if [[ "${should_check_live_seam}" == "1" ]]; then
echo ""
echo "=== Проверка ClickHouse: стык backfill/live в DDS ==="
live_pairing_query="
WITH
parseDateTime64BestEffort('${manifest_model_t_end}', 6, 'UTC') AS t_end,
t_end + INTERVAL ${GEN_LIVE_CHECK_MINUTES} MINUTE AS t_live_end,
live_browser AS (
SELECT
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id
FROM stg.browser_raw
WHERE parseDateTime64BestEffortOrNull(
JSONExtractString(raw, 'event_timestamp'), 6, 'UTC'
) >= t_end
AND parseDateTime64BestEffortOrNull(
JSONExtractString(raw, 'event_timestamp'), 6, 'UTC'
) < t_live_end
AND event_id IS NOT NULL
AND click_id IS NOT NULL
GROUP BY event_id, click_id
),
live_click_ids AS (
SELECT click_id
FROM live_browser
GROUP BY click_id
),
browser_counts AS (
SELECT click_id, count() AS browser_rows
FROM
(
SELECT
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id
FROM stg.browser_raw
WHERE event_id IS NOT NULL AND click_id IS NOT NULL
GROUP BY event_id, click_id
)
WHERE click_id IN (SELECT click_id FROM live_click_ids)
GROUP BY click_id
),
location_ids AS (
SELECT toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id
FROM stg.location_raw
WHERE event_id IS NOT NULL
GROUP BY event_id
),
device_counts AS (
SELECT
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
count() AS device_rows
FROM stg.device_raw
WHERE click_id IS NOT NULL
AND click_id IN (SELECT click_id FROM live_click_ids)
GROUP BY click_id
),
geo_counts AS (
SELECT
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
count() AS geo_rows
FROM stg.geo_raw
WHERE click_id IS NOT NULL
AND click_id IN (SELECT click_id FROM live_click_ids)
GROUP BY click_id
)
SELECT
(SELECT count() FROM live_browser) AS live_browser_rows,
(
SELECT count()
FROM live_browser AS b
LEFT JOIN location_ids AS l ON l.event_id = b.event_id
WHERE l.event_id IS NULL
) AS unpaired_location_rows,
(
SELECT ifNull(sum(if(
b.browser_rows > ifNull(d.device_rows, 0),
b.browser_rows - ifNull(d.device_rows, 0),
0
)), 0)
FROM browser_counts AS b
LEFT JOIN device_counts AS d ON d.click_id = b.click_id
) AS unpaired_device_rows,
(
SELECT ifNull(sum(if(
b.browser_rows > ifNull(g.geo_rows, 0),
b.browser_rows - ifNull(g.geo_rows, 0),
0
)), 0)
FROM browser_counts AS b
LEFT JOIN geo_counts AS g ON g.click_id = b.click_id
) AS unpaired_geo_rows
SETTINGS join_use_nulls = 1
FORMAT TabSeparated"
live_pairing="$(ch_query "${live_pairing_query}")"
IFS=$'\t' read -r live_browser_rows unpaired_location_rows unpaired_device_rows unpaired_geo_rows <<< "${live_pairing}"
[[ "${live_browser_rows}" =~ ^[0-9]+$ ]] \
&& [[ "${unpaired_location_rows}" =~ ^[0-9]+$ ]] \
&& [[ "${unpaired_device_rows}" =~ ^[0-9]+$ ]] \
&& [[ "${unpaired_geo_rows}" =~ ^[0-9]+$ ]] \
|| fail "не удалось прочитать пары live-строк"
if (( unpaired_location_rows > 0 || unpaired_device_rows > 0 || unpaired_geo_rows > 0 )); then
fail "непарные live-строки: location=${unpaired_location_rows}, device=${unpaired_device_rows}, geo=${unpaired_geo_rows} из browser=${live_browser_rows}"
fi
echo "live_browser_rows=${live_browser_rows}"
echo "unpaired_location_rows=${unpaired_location_rows}"
echo "unpaired_device_rows=${unpaired_device_rows}"
echo "unpaired_geo_rows=${unpaired_geo_rows}"
seam_duplicates_query="
WITH
toDateTime64('${CH_MODEL_T0}', 6) AS t0,
+47 -5
View File
@@ -57,9 +57,47 @@ SELECT
FORMAT TabSeparated"
}
stg_crossing_visits() {
${COMPOSE_BIN} exec -T clickhouse clickhouse-client \
--user=default \
--password=123456 \
--query "
WITH
parseDateTime64BestEffort('${GEN_MODEL_T_END}', 6, 'UTC') AS t_end,
t_end + INTERVAL ${GEN_LIVE_CHECK_MINUTES} MINUTE AS t_live_end,
browser AS (
SELECT
JSONExtractString(raw, 'click_id') AS click_id,
parseDateTime64BestEffortOrNull(
JSONExtractString(raw, 'event_timestamp'), 6, 'UTC'
) AS event_ts
FROM stg.browser_raw
WHERE click_id != ''
AND event_ts >= t_end - INTERVAL 30 MINUTE
AND event_ts < t_live_end
)
SELECT count()
FROM
(
SELECT
click_id,
min(event_ts) AS first_ts,
max(event_ts) AS last_ts
FROM browser
GROUP BY click_id
HAVING first_ts < t_end AND last_ts >= t_end
)
FORMAT TabSeparated"
}
cleanup_live_generator() {
${COMPOSE_BIN} stop generator >/dev/null 2>&1 || true
}
stop_live_generator() {
# docker compose stop возвращается только после фактической остановки контейнера.
${COMPOSE_BIN} stop generator
}
trap cleanup_live_generator EXIT
echo "=== Быстрая runtime-проверка startup-history/live seam ==="
@@ -103,7 +141,7 @@ sleep "${WAIT_STG_SECONDS}"
bash "${SCRIPT_DIR}/run_batch.sh"
stg_rows_before_live="$(stg_total_rows)"
echo "Шаг 6: live-продолжение, ждём новые STG-строки до ${LIVE_SECONDS} сек."
echo "Шаг 6: live-продолжение, ждём переходящий визит в STG до ${LIVE_SECONDS} сек."
GEN_RUN_MODE=live \
GEN_STATE_RESET=false \
GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
@@ -120,19 +158,23 @@ GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
${COMPOSE_BIN} up -d --build generator
deadline=$((SECONDS + LIVE_SECONDS))
while true; do
stg_rows_after_live="$(stg_total_rows)"
if (( stg_rows_after_live > stg_rows_before_live )); then
crossing_visits="$(stg_crossing_visits)"
[[ "${crossing_visits}" =~ ^[0-9]+$ ]] \
|| { echo "Ошибка: не удалось прочитать переходящие визиты из STG." >&2; exit 1; }
if (( crossing_visits > 0 )); then
stg_rows_after_live="$(stg_total_rows)"
echo "live_stg_rows_before=${stg_rows_before_live}"
echo "live_stg_rows_after=${stg_rows_after_live}"
echo "live_stg_crossing_visits=${crossing_visits}"
break
fi
if (( SECONDS >= deadline )); then
echo "Ошибка: live-продолжение не записало новые STG-строки за ${LIVE_SECONDS} сек." >&2
echo "Ошибка: live-продолжение не создало переходящий визит в STG за ${LIVE_SECONDS} сек." >&2
exit 1
fi
sleep 1
done
cleanup_live_generator
stop_live_generator
echo "Шаг 7: второй batch после live"
sleep "${WAIT_STG_SECONDS}"