feat(generator): добавлен артефакт стартовой истории
- Зачем: - чистый стенд должен восстанавливать стартовую историю без повторной генерации. - Что: - добавлены export/import артефакта через Kafka и compact-топики. - добавлена manifest-aware сверка ClickHouse и защита от смешения state. - добавлен runbook использования стартовой истории. - Проверка: - uv run --with-requirements generator/requirements.txt pytest generator/tests -q. - bash -n scripts/check_startup_history_manifest.sh scripts/export_startup_history_artifact.sh scripts/import_startup_history_artifact.sh. - git diff --check.
This commit is contained in:
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Сверяет DM-витрину ClickHouse с manifest портативной стартовой истории.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
CLICKHOUSE_SERVICE="${CLICKHOUSE_SERVICE:-clickhouse}"
|
||||
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
|
||||
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-123456}"
|
||||
ARTIFACT="${ARTIFACT:-/tmp/clickstream-startup-history.json}"
|
||||
|
||||
fail() {
|
||||
echo "Ошибка: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
clickhouse_datetime_literal() {
|
||||
local value="$1"
|
||||
value="${value/T/ }"
|
||||
value="${value%Z}"
|
||||
if [[ "${value}" =~ ^(.*)[+-][0-9]{2}:[0-9]{2}$ ]]; then
|
||||
value="${BASH_REMATCH[1]}"
|
||||
fi
|
||||
echo "${value}"
|
||||
}
|
||||
|
||||
[[ -s "${ARTIFACT}" ]] || fail "артефакт не найден или пуст: ${ARTIFACT}"
|
||||
|
||||
ARTIFACT_DIR="$(dirname "${ARTIFACT}")"
|
||||
ARTIFACT_FILE="$(basename "${ARTIFACT}")"
|
||||
|
||||
manifest_summary="$(${COMPOSE_BIN} run --rm --no-deps \
|
||||
-v "${ARTIFACT_DIR}:/startup-artifacts:ro" \
|
||||
generator python -m clickstream_generator.startup_history_artifact manifest-summary \
|
||||
--artifact "/startup-artifacts/${ARTIFACT_FILE}")"
|
||||
|
||||
IFS=$'\t' read -r expected_events expected_visits expected_users expected_min_ts expected_max_ts model_t0 model_t_end <<< "${manifest_summary}"
|
||||
|
||||
[[ "${expected_events}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать manifest из артефакта"
|
||||
|
||||
ch_model_t0="$(clickhouse_datetime_literal "${model_t0}")"
|
||||
ch_model_t_end="$(clickhouse_datetime_literal "${model_t_end}")"
|
||||
|
||||
actual="$(${COMPOSE_BIN} exec -T "${CLICKHOUSE_SERVICE}" clickhouse-client \
|
||||
--user="${CLICKHOUSE_USER}" \
|
||||
--password="${CLICKHOUSE_PASSWORD}" \
|
||||
--query "
|
||||
WITH
|
||||
toDateTime64('${ch_model_t0}', 6) AS t0,
|
||||
toDateTime64('${ch_model_t_end}', 6) AS t_end
|
||||
SELECT
|
||||
count() AS events,
|
||||
uniqExact(click_id) AS visits,
|
||||
uniqExact(user_domain_id) AS users,
|
||||
toString(min(event_ts)) AS min_event_timestamp,
|
||||
toString(max(event_ts)) AS max_event_timestamp
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_ts >= t0 AND event_ts < t_end
|
||||
FORMAT TabSeparated")"
|
||||
|
||||
IFS=$'\t' read -r actual_events actual_visits actual_users actual_min_ts actual_max_ts <<< "${actual}"
|
||||
|
||||
[[ "${actual_events}" == "${expected_events}" ]] || fail "events: ClickHouse=${actual_events}, manifest=${expected_events}"
|
||||
[[ "${actual_visits}" == "${expected_visits}" ]] || fail "visits: ClickHouse=${actual_visits}, manifest=${expected_visits}"
|
||||
[[ "${actual_users}" == "${expected_users}" ]] || fail "users: ClickHouse=${actual_users}, manifest=${expected_users}"
|
||||
[[ "${actual_min_ts}" == "${expected_min_ts}" ]] || fail "min_event_timestamp: ClickHouse=${actual_min_ts}, manifest=${expected_min_ts}"
|
||||
[[ "${actual_max_ts}" == "${expected_max_ts}" ]] || fail "max_event_timestamp: ClickHouse=${actual_max_ts}, manifest=${expected_max_ts}"
|
||||
|
||||
echo "manifest_events=${expected_events}"
|
||||
echo "manifest_visits=${expected_visits}"
|
||||
echo "manifest_users=${expected_users}"
|
||||
echo "manifest_min_event_timestamp=${expected_min_ts}"
|
||||
echo "manifest_max_event_timestamp=${expected_max_ts}"
|
||||
echo "ClickHouse совпадает с manifest артефакта."
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Экспортирует стартовую историю в портативный JSON-артефакт.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
CLEAN_START="${CLEAN_START:-1}"
|
||||
ARTIFACT="${ARTIFACT:-/tmp/clickstream-startup-history.json}"
|
||||
|
||||
GEN_SEED="${GEN_SEED:-4242}"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
|
||||
GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE:-UTC}"
|
||||
GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED:-1}"
|
||||
GEN_TICK_SECONDS="${GEN_TICK_SECONDS:-60}"
|
||||
GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN:-60}"
|
||||
GEN_JITTER_PCT="${GEN_JITTER_PCT:-0}"
|
||||
GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK:-1}"
|
||||
GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK:-1000}"
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
ARTIFACT_DIR="$(dirname "${ARTIFACT}")"
|
||||
ARTIFACT_FILE="$(basename "${ARTIFACT}")"
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
|
||||
echo "=== Экспорт стартовой истории ==="
|
||||
echo "ARTIFACT=${ARTIFACT}"
|
||||
echo "GEN_SEED=${GEN_SEED}"
|
||||
echo "GEN_MODEL_T0=${GEN_MODEL_T0}"
|
||||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||||
echo ""
|
||||
|
||||
if [[ "${CLEAN_START}" == "1" ]]; then
|
||||
echo "Шаг 0: очистка Kafka/state перед повторяемым экспортом"
|
||||
${COMPOSE_BIN} down -v --remove-orphans
|
||||
else
|
||||
echo "Шаг 0: CLEAN_START=0, очистка пропущена"
|
||||
fi
|
||||
|
||||
echo "Шаг 1: запуск Kafka"
|
||||
${COMPOSE_BIN} up -d kafka
|
||||
|
||||
echo "Шаг 2: сборка образа генератора"
|
||||
${COMPOSE_BIN} build generator
|
||||
|
||||
echo "Шаг 3: backfill и запись артефакта"
|
||||
${COMPOSE_BIN} run --rm --no-deps \
|
||||
-v "${ARTIFACT_DIR}:/startup-artifacts" \
|
||||
-e GEN_RUN_MODE=backfill \
|
||||
-e GEN_STATE_RESET=true \
|
||||
-e GEN_STARTUP_HISTORY_ARTIFACT="/startup-artifacts/${ARTIFACT_FILE}" \
|
||||
-e GEN_SEED="${GEN_SEED}" \
|
||||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
-e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||||
-e GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
|
||||
-e GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
|
||||
-e GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
|
||||
-e GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
|
||||
-e GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
|
||||
-e GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
|
||||
-e GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
|
||||
generator
|
||||
|
||||
test -s "${ARTIFACT}"
|
||||
echo "Готово: ${ARTIFACT}"
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Импортирует портативный артефакт стартовой истории в Kafka.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
ARTIFACT="${ARTIFACT:-/tmp/clickstream-startup-history.json}"
|
||||
|
||||
GEN_SEED="${GEN_SEED:-4242}"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
|
||||
GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE:-UTC}"
|
||||
GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED:-1}"
|
||||
GEN_TICK_SECONDS="${GEN_TICK_SECONDS:-60}"
|
||||
GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN:-60}"
|
||||
GEN_JITTER_PCT="${GEN_JITTER_PCT:-0}"
|
||||
GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK:-1}"
|
||||
GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK:-1000}"
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
if [[ ! -s "${ARTIFACT}" ]]; then
|
||||
echo "Ошибка: артефакт не найден или пуст: ${ARTIFACT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARTIFACT_DIR="$(dirname "${ARTIFACT}")"
|
||||
ARTIFACT_FILE="$(basename "${ARTIFACT}")"
|
||||
|
||||
echo "=== Импорт стартовой истории ==="
|
||||
echo "ARTIFACT=${ARTIFACT}"
|
||||
echo "GEN_SEED=${GEN_SEED}"
|
||||
echo "GEN_MODEL_T0=${GEN_MODEL_T0}"
|
||||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||||
echo ""
|
||||
|
||||
echo "Шаг 1: запуск Kafka"
|
||||
${COMPOSE_BIN} up -d kafka
|
||||
|
||||
echo "Шаг 2: сборка образа генератора"
|
||||
${COMPOSE_BIN} build generator
|
||||
|
||||
echo "Шаг 3: воспроизведение артефакта в Kafka"
|
||||
${COMPOSE_BIN} run --rm --no-deps \
|
||||
-v "${ARTIFACT_DIR}:/startup-artifacts:ro" \
|
||||
-e GEN_SEED="${GEN_SEED}" \
|
||||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
-e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||||
-e GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
|
||||
-e GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
|
||||
-e GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
|
||||
-e GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
|
||||
-e GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
|
||||
-e GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
|
||||
-e GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
|
||||
generator python -m clickstream_generator.startup_history_artifact import \
|
||||
--artifact "/startup-artifacts/${ARTIFACT_FILE}"
|
||||
|
||||
echo "Готово: артефакт воспроизведён в Kafka."
|
||||
Reference in New Issue
Block a user