feat(generator): добавлены профили запуска
- Зачем: - запуск генератора должен быть понятным перед будущим DAG-пультом. - Что: - добавлены глаголы запуска backfill, continue и reset. - добавлены профили ci и daily-wave с расчётом длительности истории. - обновлены runbook и документы запуска под профильный интерфейс. - Проверка: - uv run --with-requirements generator/requirements.txt pytest generator/tests -q. - bash -n scripts/run_generator.sh scripts/export_startup_history_artifact.sh scripts/import_startup_history_artifact.sh scripts/run_generated_history_analytics.sh. - PROFILE=daily-wave COMPOSE_BIN=true bash scripts/run_generator.sh backfill.
This commit is contained in:
@@ -10,26 +10,31 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
CLEAN_START="${CLEAN_START:-1}"
|
||||
ARTIFACT="${ARTIFACT:-/tmp/clickstream-startup-history.json}"
|
||||
|
||||
GEN_SEED="${GEN_SEED:-4242}"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
|
||||
GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE:-UTC}"
|
||||
GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED:-1}"
|
||||
GEN_TICK_SECONDS="${GEN_TICK_SECONDS:-60}"
|
||||
GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN:-60}"
|
||||
GEN_JITTER_PCT="${GEN_JITTER_PCT:-0}"
|
||||
GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK:-1}"
|
||||
GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK:-1000}"
|
||||
PROFILE="${PROFILE:-${GEN_LAUNCH_PROFILE:-ci}}"
|
||||
duration_args=()
|
||||
if [[ -n "${GEN_HISTORY_DURATION:-}" ]]; then
|
||||
duration_args=(--duration "${GEN_HISTORY_DURATION}")
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
launch_env="$(
|
||||
PYTHONPATH="${REPO_ROOT}/generator/src" \
|
||||
uv run python -m clickstream_generator.launch \
|
||||
backfill \
|
||||
--profile "${PROFILE}" \
|
||||
"${duration_args[@]}"
|
||||
)"
|
||||
eval "${launch_env}"
|
||||
|
||||
ARTIFACT_DIR="$(dirname "${ARTIFACT}")"
|
||||
ARTIFACT_FILE="$(basename "${ARTIFACT}")"
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
|
||||
echo "=== Экспорт стартовой истории ==="
|
||||
echo "ARTIFACT=${ARTIFACT}"
|
||||
echo "GEN_LAUNCH_PROFILE=${GEN_LAUNCH_PROFILE}"
|
||||
echo "GEN_HISTORY_DURATION=${GEN_HISTORY_DURATION}"
|
||||
echo "GEN_SEED=${GEN_SEED}"
|
||||
echo "GEN_MODEL_T0=${GEN_MODEL_T0}"
|
||||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||||
@@ -53,6 +58,7 @@ ${COMPOSE_BIN} run --rm --no-deps \
|
||||
-v "${ARTIFACT_DIR}:/startup-artifacts" \
|
||||
-e GEN_RUN_MODE=backfill \
|
||||
-e GEN_STATE_RESET=true \
|
||||
-e GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
|
||||
-e GEN_STARTUP_HISTORY_ARTIFACT="/startup-artifacts/${ARTIFACT_FILE}" \
|
||||
-e GEN_SEED="${GEN_SEED}" \
|
||||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
|
||||
@@ -9,20 +9,23 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
ARTIFACT="${ARTIFACT:-/tmp/clickstream-startup-history.json}"
|
||||
|
||||
GEN_SEED="${GEN_SEED:-4242}"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
|
||||
GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE:-UTC}"
|
||||
GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED:-1}"
|
||||
GEN_TICK_SECONDS="${GEN_TICK_SECONDS:-60}"
|
||||
GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN:-60}"
|
||||
GEN_JITTER_PCT="${GEN_JITTER_PCT:-0}"
|
||||
GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK:-1}"
|
||||
GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK:-1000}"
|
||||
PROFILE="${PROFILE:-${GEN_LAUNCH_PROFILE:-ci}}"
|
||||
duration_args=()
|
||||
if [[ -n "${GEN_HISTORY_DURATION:-}" ]]; then
|
||||
duration_args=(--duration "${GEN_HISTORY_DURATION}")
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
launch_env="$(
|
||||
PYTHONPATH="${REPO_ROOT}/generator/src" \
|
||||
uv run python -m clickstream_generator.launch \
|
||||
continue \
|
||||
--profile "${PROFILE}" \
|
||||
"${duration_args[@]}"
|
||||
)"
|
||||
eval "${launch_env}"
|
||||
|
||||
if [[ ! -s "${ARTIFACT}" ]]; then
|
||||
echo "Ошибка: артефакт не найден или пуст: ${ARTIFACT}" >&2
|
||||
exit 1
|
||||
@@ -33,6 +36,8 @@ ARTIFACT_FILE="$(basename "${ARTIFACT}")"
|
||||
|
||||
echo "=== Импорт стартовой истории ==="
|
||||
echo "ARTIFACT=${ARTIFACT}"
|
||||
echo "GEN_LAUNCH_PROFILE=${GEN_LAUNCH_PROFILE}"
|
||||
echo "GEN_HISTORY_DURATION=${GEN_HISTORY_DURATION}"
|
||||
echo "GEN_SEED=${GEN_SEED}"
|
||||
echo "GEN_MODEL_T0=${GEN_MODEL_T0}"
|
||||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||||
@@ -47,6 +52,7 @@ ${COMPOSE_BIN} build generator
|
||||
echo "Шаг 3: воспроизведение артефакта в Kafka"
|
||||
${COMPOSE_BIN} run --rm --no-deps \
|
||||
-v "${ARTIFACT_DIR}:/startup-artifacts:ro" \
|
||||
-e GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
|
||||
-e GEN_SEED="${GEN_SEED}" \
|
||||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
-e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||||
|
||||
@@ -15,20 +15,23 @@ COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
CLEAN_START="${CLEAN_START:-1}"
|
||||
WAIT_CLICKHOUSE_SECONDS="${WAIT_CLICKHOUSE_SECONDS:-60}"
|
||||
WAIT_STG_SECONDS="${WAIT_STG_SECONDS:-10}"
|
||||
|
||||
GEN_SEED="${GEN_SEED:-4242}"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
|
||||
GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE:-UTC}"
|
||||
GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED:-1}"
|
||||
GEN_TICK_SECONDS="${GEN_TICK_SECONDS:-60}"
|
||||
GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN:-60}"
|
||||
GEN_JITTER_PCT="${GEN_JITTER_PCT:-0}"
|
||||
GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK:-1}"
|
||||
GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK:-1000}"
|
||||
PROFILE="${PROFILE:-${GEN_LAUNCH_PROFILE:-ci}}"
|
||||
duration_args=()
|
||||
if [[ -n "${GEN_HISTORY_DURATION:-}" ]]; then
|
||||
duration_args=(--duration "${GEN_HISTORY_DURATION}")
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
launch_env="$(
|
||||
PYTHONPATH="${REPO_ROOT}/generator/src" \
|
||||
uv run python -m clickstream_generator.launch \
|
||||
backfill \
|
||||
--profile "${PROFILE}" \
|
||||
"${duration_args[@]}"
|
||||
)"
|
||||
eval "${launch_env}"
|
||||
|
||||
wait_for_clickhouse() {
|
||||
local deadline
|
||||
deadline=$((SECONDS + WAIT_CLICKHOUSE_SECONDS))
|
||||
@@ -46,6 +49,8 @@ wait_for_clickhouse() {
|
||||
}
|
||||
|
||||
echo "=== Чистый прогон стартовой истории как источника аналитики ==="
|
||||
echo "GEN_LAUNCH_PROFILE=${GEN_LAUNCH_PROFILE}"
|
||||
echo "GEN_HISTORY_DURATION=${GEN_HISTORY_DURATION}"
|
||||
echo "GEN_SEED=${GEN_SEED}"
|
||||
echo "GEN_MODEL_T0=${GEN_MODEL_T0}"
|
||||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||||
@@ -73,6 +78,7 @@ echo "Шаг 4: backfill стартовой истории в Kafka"
|
||||
${COMPOSE_BIN} run --rm --no-deps \
|
||||
-e GEN_RUN_MODE=backfill \
|
||||
-e GEN_STATE_RESET=true \
|
||||
-e GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
|
||||
-e GEN_SEED="${GEN_SEED}" \
|
||||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
-e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Запускает генератор через понятные глаголы поверх старых env-переменных.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
echo "Использование: $0 backfill|continue|reset" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
VERB="$1"
|
||||
case "${VERB}" in
|
||||
backfill|continue|reset) ;;
|
||||
*)
|
||||
echo "Ошибка: неизвестный глагол запуска: ${VERB}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
PROFILE="${PROFILE:-${GEN_LAUNCH_PROFILE:-ci}}"
|
||||
duration_args=()
|
||||
if [[ -n "${GEN_HISTORY_DURATION:-}" ]]; then
|
||||
duration_args=(--duration "${GEN_HISTORY_DURATION}")
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
launch_env="$(
|
||||
PYTHONPATH="${REPO_ROOT}/generator/src" \
|
||||
uv run python -m clickstream_generator.launch \
|
||||
"${VERB}" \
|
||||
--profile "${PROFILE}" \
|
||||
"${duration_args[@]}"
|
||||
)"
|
||||
eval "${launch_env}"
|
||||
|
||||
echo "=== Запуск генератора: ${GEN_LAUNCH_VERB} ==="
|
||||
echo "GEN_LAUNCH_PROFILE=${GEN_LAUNCH_PROFILE}"
|
||||
if [[ -n "${GEN_HISTORY_DURATION:-}" ]]; then
|
||||
echo "GEN_HISTORY_DURATION=${GEN_HISTORY_DURATION}"
|
||||
fi
|
||||
echo "GEN_RUN_MODE=${GEN_RUN_MODE}"
|
||||
echo "GEN_STATE_RESET=${GEN_STATE_RESET}"
|
||||
if [[ -n "${GEN_MODEL_T_END:-}" ]]; then
|
||||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [[ "${VERB}" == "backfill" ]]; then
|
||||
echo "Шаг 1: запуск Kafka"
|
||||
${COMPOSE_BIN} up -d kafka
|
||||
|
||||
echo "Шаг 2: сборка образа генератора"
|
||||
${COMPOSE_BIN} build generator
|
||||
|
||||
echo "Шаг 3: backfill стартовой истории"
|
||||
${COMPOSE_BIN} run --rm --no-deps \
|
||||
-e GEN_RUN_MODE="${GEN_RUN_MODE}" \
|
||||
-e GEN_STATE_RESET="${GEN_STATE_RESET}" \
|
||||
-e GEN_LAUNCH_PROFILE="${GEN_LAUNCH_PROFILE}" \
|
||||
-e GEN_SEED="${GEN_SEED}" \
|
||||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
-e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||||
-e GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
|
||||
-e GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
|
||||
-e GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
|
||||
-e GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
|
||||
-e GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
|
||||
-e GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
|
||||
-e GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
|
||||
generator
|
||||
else
|
||||
echo "Шаг 1: запуск live-генератора"
|
||||
${COMPOSE_BIN} up -d --build generator
|
||||
fi
|
||||
Reference in New Issue
Block a user