feat(generator): добавлен артефакт стартовой истории

- Зачем:
  - чистый стенд должен восстанавливать стартовую историю без повторной генерации.
- Что:
  - добавлены export/import артефакта через Kafka и compact-топики.
  - добавлена manifest-aware сверка ClickHouse и защита от смешения state.
  - добавлен runbook использования стартовой истории.
- Проверка:
  - uv run --with-requirements generator/requirements.txt pytest generator/tests -q.
  - bash -n scripts/check_startup_history_manifest.sh scripts/export_startup_history_artifact.sh scripts/import_startup_history_artifact.sh.
  - git diff --check.
This commit is contained in:
2026-07-04 18:23:54 +03:00
parent ee4c9f5857
commit 3c38465d89
16 changed files with 1746 additions and 177 deletions
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
#
# Сверяет DM-витрину ClickHouse с manifest портативной стартовой истории.
set -euo pipefail
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
CLICKHOUSE_SERVICE="${CLICKHOUSE_SERVICE:-clickhouse}"
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-123456}"
ARTIFACT="${ARTIFACT:-/tmp/clickstream-startup-history.json}"
fail() {
echo "Ошибка: $*" >&2
exit 1
}
clickhouse_datetime_literal() {
local value="$1"
value="${value/T/ }"
value="${value%Z}"
if [[ "${value}" =~ ^(.*)[+-][0-9]{2}:[0-9]{2}$ ]]; then
value="${BASH_REMATCH[1]}"
fi
echo "${value}"
}
[[ -s "${ARTIFACT}" ]] || fail "артефакт не найден или пуст: ${ARTIFACT}"
ARTIFACT_DIR="$(dirname "${ARTIFACT}")"
ARTIFACT_FILE="$(basename "${ARTIFACT}")"
manifest_summary="$(${COMPOSE_BIN} run --rm --no-deps \
-v "${ARTIFACT_DIR}:/startup-artifacts:ro" \
generator python -m clickstream_generator.startup_history_artifact manifest-summary \
--artifact "/startup-artifacts/${ARTIFACT_FILE}")"
IFS=$'\t' read -r expected_events expected_visits expected_users expected_min_ts expected_max_ts model_t0 model_t_end <<< "${manifest_summary}"
[[ "${expected_events}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать manifest из артефакта"
ch_model_t0="$(clickhouse_datetime_literal "${model_t0}")"
ch_model_t_end="$(clickhouse_datetime_literal "${model_t_end}")"
actual="$(${COMPOSE_BIN} exec -T "${CLICKHOUSE_SERVICE}" clickhouse-client \
--user="${CLICKHOUSE_USER}" \
--password="${CLICKHOUSE_PASSWORD}" \
--query "
WITH
toDateTime64('${ch_model_t0}', 6) AS t0,
toDateTime64('${ch_model_t_end}', 6) AS t_end
SELECT
count() AS events,
uniqExact(click_id) AS visits,
uniqExact(user_domain_id) AS users,
toString(min(event_ts)) AS min_event_timestamp,
toString(max(event_ts)) AS max_event_timestamp
FROM dm.v_events_enriched
WHERE event_ts >= t0 AND event_ts < t_end
FORMAT TabSeparated")"
IFS=$'\t' read -r actual_events actual_visits actual_users actual_min_ts actual_max_ts <<< "${actual}"
[[ "${actual_events}" == "${expected_events}" ]] || fail "events: ClickHouse=${actual_events}, manifest=${expected_events}"
[[ "${actual_visits}" == "${expected_visits}" ]] || fail "visits: ClickHouse=${actual_visits}, manifest=${expected_visits}"
[[ "${actual_users}" == "${expected_users}" ]] || fail "users: ClickHouse=${actual_users}, manifest=${expected_users}"
[[ "${actual_min_ts}" == "${expected_min_ts}" ]] || fail "min_event_timestamp: ClickHouse=${actual_min_ts}, manifest=${expected_min_ts}"
[[ "${actual_max_ts}" == "${expected_max_ts}" ]] || fail "max_event_timestamp: ClickHouse=${actual_max_ts}, manifest=${expected_max_ts}"
echo "manifest_events=${expected_events}"
echo "manifest_visits=${expected_visits}"
echo "manifest_users=${expected_users}"
echo "manifest_min_event_timestamp=${expected_min_ts}"
echo "manifest_max_event_timestamp=${expected_max_ts}"
echo "ClickHouse совпадает с manifest артефакта."