feat(stand): переведён аналитический путь на стартовую историю
- Зачем: - чистый стенд должен строить аналитику из генерации, а не из архивного JSONL-сида. - Что: - добавлены команды generated-history-analytics и generated-history-check. - обновлены README, operations и Superset-документы под путь generator backfill -> DM -> Superset. - создан follow-up на миграцию учебных материалов с архивного сида. - Проверка: - make generated-history-analytics. - make generated-history-check. - reviewer gate issue 06 пройден без блокирующих находок.
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Повторяемая проверка, что DM-витрины и Superset metadata работают на данных
|
||||
# стартовой истории генератора, а не на архивном сиде 2022 года.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
CLICKHOUSE_SERVICE="${CLICKHOUSE_SERVICE:-clickhouse}"
|
||||
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
|
||||
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-123456}"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
|
||||
REQUIRE_SUPERSET="${REQUIRE_SUPERSET:-1}"
|
||||
|
||||
fail() {
|
||||
echo "Ошибка: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
clickhouse_datetime_literal() {
|
||||
local value="$1"
|
||||
value="${value/T/ }"
|
||||
value="${value%Z}"
|
||||
if [[ "${value}" =~ ^(.*)[+-][0-9]{2}:[0-9]{2}$ ]]; then
|
||||
value="${BASH_REMATCH[1]}"
|
||||
fi
|
||||
echo "${value}"
|
||||
}
|
||||
|
||||
ch_query() {
|
||||
${COMPOSE_BIN} exec -T "${CLICKHOUSE_SERVICE}" clickhouse-client \
|
||||
--user="${CLICKHOUSE_USER}" \
|
||||
--password="${CLICKHOUSE_PASSWORD}" \
|
||||
--query "$1"
|
||||
}
|
||||
|
||||
pg_query() {
|
||||
${COMPOSE_BIN} exec -T postgres-metadata psql \
|
||||
-U airflow \
|
||||
-d superset \
|
||||
-At \
|
||||
-c "$1" | tr -d '\r'
|
||||
}
|
||||
|
||||
CH_MODEL_T0="$(clickhouse_datetime_literal "${GEN_MODEL_T0}")"
|
||||
CH_MODEL_T_END="$(clickhouse_datetime_literal "${GEN_MODEL_T_END}")"
|
||||
|
||||
echo "=== Проверка ClickHouse: данные генерации в DM ==="
|
||||
|
||||
stats_query="
|
||||
WITH
|
||||
toDateTime64('${CH_MODEL_T0}', 6) AS t0,
|
||||
toDateTime64('${CH_MODEL_T_END}', 6) AS t_end
|
||||
SELECT
|
||||
count() AS events,
|
||||
uniqExact(click_id) AS visits,
|
||||
uniqExact(user_domain_id) AS users,
|
||||
toString(min(event_ts)) AS min_event_ts,
|
||||
toString(max(event_ts)) AS max_event_ts,
|
||||
users < visits AND visits < events AS pyramid_ok,
|
||||
min(event_ts) >= t0 AND max(event_ts) < t_end AS half_open_ok,
|
||||
countIf(toYear(event_ts) = 2022) = 0 AS no_2022_rows,
|
||||
hex(sipHash128(groupArray(tuple(event_id, click_id, user_domain_id, event_ts, page_url_path)))) AS digest
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
event_id,
|
||||
click_id,
|
||||
user_domain_id,
|
||||
event_ts,
|
||||
page_url_path
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_ts >= t0 AND event_ts < t_end
|
||||
ORDER BY event_id
|
||||
)
|
||||
FORMAT TabSeparated"
|
||||
|
||||
stats="$(ch_query "${stats_query}")"
|
||||
IFS=$'\t' read -r events visits users min_event_ts max_event_ts pyramid_ok half_open_ok no_2022_rows digest <<< "${stats}"
|
||||
|
||||
[[ "${events}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать число событий из dm.v_events_enriched"
|
||||
(( events > 0 )) || fail "dm.v_events_enriched пустая в модельном диапазоне"
|
||||
[[ "${pyramid_ok}" == "1" ]] || fail "нарушена пирамида users < visits < events"
|
||||
[[ "${half_open_ok}" == "1" ]] || fail "данные вышли за диапазон [GEN_MODEL_T0, GEN_MODEL_T_END)"
|
||||
[[ "${no_2022_rows}" == "1" ]] || fail "найдены строки 2022 года, похожие на архивный сид"
|
||||
|
||||
echo "events=${events}"
|
||||
echo "visits=${visits}"
|
||||
echo "users=${users}"
|
||||
echo "min_event_ts=${min_event_ts}"
|
||||
echo "max_event_ts=${max_event_ts}"
|
||||
echo "digest=${digest}"
|
||||
|
||||
echo ""
|
||||
echo "=== Проверка ClickHouse: возвраты пользователей ==="
|
||||
|
||||
returns_query="
|
||||
WITH
|
||||
toDateTime64('${CH_MODEL_T0}', 6) AS t0,
|
||||
toDateTime64('${CH_MODEL_T_END}', 6) AS t_end,
|
||||
users AS (
|
||||
SELECT user_domain_id, uniqExact(click_id) AS visits
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_ts >= t0 AND event_ts < t_end
|
||||
AND user_domain_id IS NOT NULL
|
||||
GROUP BY user_domain_id
|
||||
)
|
||||
SELECT
|
||||
count() AS users,
|
||||
countIf(visits > 1) AS returning_users,
|
||||
returning_users / users AS returning_share,
|
||||
max(visits) AS max_visits_per_user
|
||||
FROM users
|
||||
FORMAT TabSeparated"
|
||||
|
||||
returns="$(ch_query "${returns_query}")"
|
||||
IFS=$'\t' read -r total_users returning_users returning_share max_visits_per_user <<< "${returns}"
|
||||
|
||||
[[ "${total_users}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать число пользователей"
|
||||
(( total_users > 0 )) || fail "нет пользователей для проверки возвратов"
|
||||
(( returning_users > 0 )) || fail "нет пользователей с повторными визитами"
|
||||
|
||||
echo "users=${total_users}"
|
||||
echo "returning_users=${returning_users}"
|
||||
echo "returning_share=${returning_share}"
|
||||
echo "max_visits_per_user=${max_visits_per_user}"
|
||||
|
||||
echo ""
|
||||
echo "=== Проверка ClickHouse: форма длины визита ==="
|
||||
|
||||
visit_shape_query="
|
||||
WITH
|
||||
toDateTime64('${CH_MODEL_T0}', 6) AS t0,
|
||||
toDateTime64('${CH_MODEL_T_END}', 6) AS t_end,
|
||||
30 AS max_session_events,
|
||||
sessions AS (
|
||||
SELECT
|
||||
click_id,
|
||||
count() AS events_count,
|
||||
dateDiff('second', min(event_ts), max(event_ts)) AS duration_sec
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_ts >= t0 AND event_ts < t_end
|
||||
GROUP BY click_id
|
||||
)
|
||||
SELECT
|
||||
count() AS visits,
|
||||
countIf(events_count <= 2) / visits AS short_visit_share,
|
||||
quantileExact(0.5)(events_count) AS median_events_per_visit,
|
||||
avg(events_count) AS avg_events_per_visit,
|
||||
countIf(events_count = max_session_events) / visits AS capped_visit_share,
|
||||
quantileExact(0.5)(duration_sec) AS median_duration_sec,
|
||||
quantileExact(0.95)(duration_sec) AS p95_duration_sec,
|
||||
max(events_count) AS max_events_per_visit
|
||||
FROM sessions
|
||||
FORMAT TabSeparated"
|
||||
|
||||
visit_shape="$(ch_query "${visit_shape_query}")"
|
||||
IFS=$'\t' read -r shape_visits short_visit_share median_events_per_visit avg_events_per_visit capped_visit_share median_duration_sec p95_duration_sec max_events_per_visit <<< "${visit_shape}"
|
||||
|
||||
[[ "${shape_visits}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать число визитов"
|
||||
(( shape_visits > 0 )) || fail "нет визитов для проверки формы"
|
||||
(( max_events_per_visit <= 30 )) || fail "длина визита превысила GEN_MAX_SESSION_EVENTS"
|
||||
|
||||
echo "visits=${shape_visits}"
|
||||
echo "short_visit_share=${short_visit_share}"
|
||||
echo "median_events_per_visit=${median_events_per_visit}"
|
||||
echo "avg_events_per_visit=${avg_events_per_visit}"
|
||||
echo "capped_visit_share=${capped_visit_share}"
|
||||
echo "median_duration_sec=${median_duration_sec}"
|
||||
echo "p95_duration_sec=${p95_duration_sec}"
|
||||
echo "max_events_per_visit=${max_events_per_visit}"
|
||||
|
||||
echo ""
|
||||
echo "=== Проверка ClickHouse: ordered funnel ==="
|
||||
|
||||
ordered_funnel_query="
|
||||
WITH
|
||||
toDateTime64('${CH_MODEL_T0}', 6) AS t0,
|
||||
toDateTime64('${CH_MODEL_T_END}', 6) AS t_end,
|
||||
sessions AS (
|
||||
SELECT
|
||||
click_id,
|
||||
minIf(event_ts, page_url_path = '/home') AS home_ts,
|
||||
minIf(event_ts, page_url_path IN ('/product_a', '/product_b')) AS product_ts,
|
||||
minIf(event_ts, page_url_path = '/cart') AS cart_ts,
|
||||
minIf(event_ts, page_url_path = '/payment') AS payment_ts,
|
||||
minIf(event_ts, page_url_path = '/confirmation') AS confirmation_ts
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_ts >= t0 AND event_ts < t_end
|
||||
GROUP BY click_id
|
||||
)
|
||||
SELECT
|
||||
countIf(home_ts IS NOT NULL) AS home,
|
||||
countIf(home_ts IS NOT NULL AND product_ts > home_ts) AS products,
|
||||
countIf(home_ts IS NOT NULL AND product_ts > home_ts AND cart_ts > product_ts) AS cart,
|
||||
countIf(home_ts IS NOT NULL AND product_ts > home_ts AND cart_ts > product_ts AND payment_ts > cart_ts) AS payment,
|
||||
countIf(home_ts IS NOT NULL AND product_ts > home_ts AND cart_ts > product_ts AND payment_ts > cart_ts AND confirmation_ts > payment_ts) AS confirmation,
|
||||
products <= home AND cart <= products AND payment <= cart AND confirmation <= payment AS monotonic_ok,
|
||||
confirmation / home AS confirmation_share
|
||||
FROM sessions
|
||||
FORMAT TabSeparated"
|
||||
|
||||
ordered_funnel="$(ch_query "${ordered_funnel_query}")"
|
||||
IFS=$'\t' read -r ordered_home ordered_products ordered_cart ordered_payment ordered_confirmation ordered_monotonic_ok ordered_confirmation_share <<< "${ordered_funnel}"
|
||||
|
||||
[[ "${ordered_home}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать ordered funnel"
|
||||
(( ordered_home > 0 )) || fail "ordered funnel: нет /home"
|
||||
[[ "${ordered_monotonic_ok}" == "1" ]] || fail "ordered funnel не монотонен"
|
||||
|
||||
echo "home=${ordered_home}"
|
||||
echo "products=${ordered_products}"
|
||||
echo "cart=${ordered_cart}"
|
||||
echo "payment=${ordered_payment}"
|
||||
echo "confirmation=${ordered_confirmation}"
|
||||
echo "monotonic_ok=${ordered_monotonic_ok}"
|
||||
echo "confirmation_share=${ordered_confirmation_share}"
|
||||
|
||||
echo ""
|
||||
echo "=== Проверка ClickHouse: contains funnel ==="
|
||||
|
||||
contains_funnel_query="
|
||||
WITH
|
||||
toDateTime64('${CH_MODEL_T0}', 6) AS t0,
|
||||
toDateTime64('${CH_MODEL_T_END}', 6) AS t_end,
|
||||
sessions AS (
|
||||
SELECT
|
||||
click_id,
|
||||
countIf(page_url_path = '/home') > 0 AS has_home,
|
||||
countIf(page_url_path IN ('/product_a', '/product_b')) > 0 AS has_product,
|
||||
countIf(page_url_path = '/cart') > 0 AS has_cart,
|
||||
countIf(page_url_path = '/payment') > 0 AS has_payment,
|
||||
countIf(page_url_path = '/confirmation') > 0 AS has_confirmation
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_ts >= t0 AND event_ts < t_end
|
||||
GROUP BY click_id
|
||||
)
|
||||
SELECT
|
||||
countIf(has_home) AS home,
|
||||
countIf(has_home AND has_product) AS products,
|
||||
countIf(has_home AND has_product AND has_cart) AS cart,
|
||||
countIf(has_home AND has_product AND has_cart AND has_payment) AS payment,
|
||||
countIf(has_home AND has_product AND has_cart AND has_payment AND has_confirmation) AS confirmation,
|
||||
products <= home AND cart <= products AND payment <= cart AND confirmation <= payment AS monotonic_ok,
|
||||
confirmation / home AS confirmation_share
|
||||
FROM sessions
|
||||
FORMAT TabSeparated"
|
||||
|
||||
contains_funnel="$(ch_query "${contains_funnel_query}")"
|
||||
IFS=$'\t' read -r contains_home contains_products contains_cart contains_payment contains_confirmation contains_monotonic_ok contains_confirmation_share <<< "${contains_funnel}"
|
||||
|
||||
[[ "${contains_home}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать contains funnel"
|
||||
(( contains_home > 0 )) || fail "contains funnel: нет /home"
|
||||
[[ "${contains_monotonic_ok}" == "1" ]] || fail "contains funnel не монотонен"
|
||||
|
||||
echo "home=${contains_home}"
|
||||
echo "products=${contains_products}"
|
||||
echo "cart=${contains_cart}"
|
||||
echo "payment=${contains_payment}"
|
||||
echo "confirmation=${contains_confirmation}"
|
||||
echo "monotonic_ok=${contains_monotonic_ok}"
|
||||
echo "confirmation_share=${contains_confirmation_share}"
|
||||
|
||||
echo ""
|
||||
echo "=== Проверка ClickHouse: основные DM-витрины не пустые ==="
|
||||
|
||||
views_query="
|
||||
SELECT source, rows
|
||||
FROM
|
||||
(
|
||||
SELECT 'dm.v_events_enriched' AS source, count() AS rows FROM dm.v_events_enriched
|
||||
UNION ALL SELECT 'dm.v_daily_traffic', count() FROM dm.v_daily_traffic
|
||||
UNION ALL SELECT 'dm.v_top_pages_daily', count() FROM dm.v_top_pages_daily
|
||||
UNION ALL SELECT 'dm.v_utm_effectiveness', count() FROM dm.v_utm_effectiveness
|
||||
UNION ALL SELECT 'dm.v_session_overview', count() FROM dm.v_session_overview
|
||||
UNION ALL SELECT 'dm.dq_summary', count() FROM dm.dq_summary
|
||||
)
|
||||
ORDER BY source
|
||||
FORMAT TabSeparated"
|
||||
|
||||
while IFS=$'\t' read -r source rows; do
|
||||
[[ -n "${source}" ]] || continue
|
||||
[[ "${rows}" =~ ^[0-9]+$ ]] || fail "не удалось прочитать число строк для ${source}"
|
||||
(( rows > 0 )) || fail "${source} пустая"
|
||||
echo "${source}=${rows}"
|
||||
done < <(ch_query "${views_query}")
|
||||
|
||||
if [[ "${REQUIRE_SUPERSET}" != "1" ]]; then
|
||||
echo ""
|
||||
echo "Проверка Superset пропущена: REQUIRE_SUPERSET=${REQUIRE_SUPERSET}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Проверка Superset: datasets, charts и dashboard созданы ==="
|
||||
|
||||
if ! ${COMPOSE_BIN} ps --services --filter "status=running" | grep -qx "superset"; then
|
||||
fail "сервис superset не запущен"
|
||||
fi
|
||||
|
||||
${COMPOSE_BIN} exec -T superset curl -fsS http://localhost:8088/health >/dev/null \
|
||||
|| fail "Superset health endpoint не отвечает"
|
||||
|
||||
dataset_count="$(pg_query "
|
||||
SELECT count(*)
|
||||
FROM tables
|
||||
WHERE schema = 'dm'
|
||||
AND table_name IN (
|
||||
'v_events_enriched',
|
||||
'v_daily_traffic',
|
||||
'v_utm_effectiveness',
|
||||
'v_top_pages_daily',
|
||||
'v_session_overview',
|
||||
'dq_summary'
|
||||
);")"
|
||||
|
||||
dashboard_count="$(pg_query "
|
||||
SELECT count(*)
|
||||
FROM dashboards
|
||||
WHERE slug = 'ecommerce-analytics';")"
|
||||
|
||||
chart_count="$(pg_query "
|
||||
SELECT count(*)
|
||||
FROM dashboard_slices ds
|
||||
JOIN dashboards d ON d.id = ds.dashboard_id
|
||||
WHERE d.slug = 'ecommerce-analytics';")"
|
||||
|
||||
[[ "${dataset_count}" == "6" ]] || fail "ожидалось 6 Superset datasets, найдено ${dataset_count}"
|
||||
[[ "${dashboard_count}" == "1" ]] || fail "dashboard ecommerce-analytics не найден"
|
||||
(( chart_count > 0 )) || fail "dashboard ecommerce-analytics не связан с chart"
|
||||
|
||||
echo "superset_datasets=${dataset_count}"
|
||||
echo "superset_dashboards=${dashboard_count}"
|
||||
echo "superset_dashboard_charts=${chart_count}"
|
||||
echo "dashboard_url=http://localhost:8088/superset/dashboard/ecommerce-analytics/"
|
||||
|
||||
echo ""
|
||||
echo "=== Проверка Superset: dashboard открывается и читает ClickHouse ==="
|
||||
|
||||
superset_probe="$(
|
||||
${COMPOSE_BIN} exec -T superset bash -s <<'PY'
|
||||
set -euo pipefail
|
||||
python - <<'PYTHON'
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from superset.app import create_app
|
||||
|
||||
app = create_app()
|
||||
with app.app_context():
|
||||
from superset.extensions import db
|
||||
from superset.models.core import Database
|
||||
from superset.models.dashboard import Dashboard
|
||||
|
||||
dashboard = (
|
||||
db.session.query(Dashboard)
|
||||
.filter_by(slug="ecommerce-analytics")
|
||||
.one()
|
||||
)
|
||||
|
||||
database = (
|
||||
db.session.query(Database)
|
||||
.filter_by(database_name="clickhouse_dwh")
|
||||
.one()
|
||||
)
|
||||
with database.get_sqla_engine() as engine:
|
||||
with engine.connect() as connection:
|
||||
events = connection.execute(
|
||||
text("SELECT count() FROM dm.v_events_enriched")
|
||||
).scalar()
|
||||
|
||||
login_payload = json.dumps(
|
||||
{
|
||||
"username": "admin",
|
||||
"password": "admin",
|
||||
"provider": "db",
|
||||
"refresh": True,
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
login_request = urllib.request.Request(
|
||||
"http://localhost:8088/api/v1/security/login",
|
||||
data=login_payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(login_request, timeout=10) as response:
|
||||
login_status = response.status
|
||||
token = json.loads(response.read().decode("utf-8"))["access_token"]
|
||||
|
||||
dashboard_request = urllib.request.Request(
|
||||
f"http://localhost:8088/api/v1/dashboard/{dashboard.id}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
method="GET",
|
||||
)
|
||||
with urllib.request.urlopen(dashboard_request, timeout=10) as response:
|
||||
dashboard_status = response.status
|
||||
dashboard_payload = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
dashboard_title = dashboard_payload["result"]["dashboard_title"]
|
||||
|
||||
print(f"login_api_status={login_status}")
|
||||
print(f"dashboard_api_status={dashboard_status}")
|
||||
print(f"dashboard_title={dashboard_title}")
|
||||
print(f"superset_clickhouse_events={events}")
|
||||
|
||||
if login_status >= 400:
|
||||
raise SystemExit("Superset login API failed")
|
||||
if dashboard_status >= 400:
|
||||
raise SystemExit("Superset dashboard API did not open")
|
||||
if "E-commerce Analytics" not in dashboard_title:
|
||||
raise SystemExit("Unexpected dashboard title")
|
||||
if not isinstance(events, int) or events <= 0:
|
||||
raise SystemExit("Superset ClickHouse query returned no data")
|
||||
PYTHON
|
||||
PY
|
||||
)"
|
||||
|
||||
echo "${superset_probe}"
|
||||
@@ -14,7 +14,7 @@
|
||||
#
|
||||
# Требования:
|
||||
# - ClickHouse запущен (make up)
|
||||
# - STG содержит данные (make data выполнен)
|
||||
# - STG содержит данные генератора или ручной загрузки
|
||||
#
|
||||
# Стратегия:
|
||||
# Сейчас: полная перезагрузка (TRUNCATE + INSERT) — для демо
|
||||
@@ -81,7 +81,8 @@ STG_COUNT=$(${COMPOSE_BIN} exec -T "${CLICKHOUSE_SERVICE}" clickhouse-client \
|
||||
|
||||
if [[ "${STG_COUNT}" == "0" ]]; then
|
||||
echo "Предупреждение: Таблицы STG пусты."
|
||||
echo "Сначала загрузите данные: make data"
|
||||
echo "Сначала загрузите данные: make generated-history-analytics"
|
||||
echo "Для ручной отладки можно выполнить backfill генератора или архивный make data."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Чистый прогон аналитического контура от стартовой истории генератора до DM и Superset.
|
||||
#
|
||||
# Команда намеренно очищает volumes по умолчанию: так сбрасываются ClickHouse,
|
||||
# Kafka-топики данных, state и manifest генератора. Это штатный повторяемый путь
|
||||
# для проверки ADR-0006 на свежем стенде.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
COMPOSE_BIN="${COMPOSE_BIN:-docker compose}"
|
||||
CLEAN_START="${CLEAN_START:-1}"
|
||||
WAIT_CLICKHOUSE_SECONDS="${WAIT_CLICKHOUSE_SECONDS:-60}"
|
||||
WAIT_STG_SECONDS="${WAIT_STG_SECONDS:-10}"
|
||||
|
||||
GEN_SEED="${GEN_SEED:-4242}"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}"
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END:-2026-01-01T06:00:00+00:00}"
|
||||
GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE:-UTC}"
|
||||
GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED:-1}"
|
||||
GEN_TICK_SECONDS="${GEN_TICK_SECONDS:-60}"
|
||||
GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN:-60}"
|
||||
GEN_JITTER_PCT="${GEN_JITTER_PCT:-0}"
|
||||
GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK:-1}"
|
||||
GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK:-1000}"
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
wait_for_clickhouse() {
|
||||
local deadline
|
||||
deadline=$((SECONDS + WAIT_CLICKHOUSE_SECONDS))
|
||||
|
||||
until ${COMPOSE_BIN} exec -T clickhouse clickhouse-client \
|
||||
--user=default \
|
||||
--password=123456 \
|
||||
--query "SELECT 1" >/dev/null 2>&1; do
|
||||
if (( SECONDS >= deadline )); then
|
||||
echo "Ошибка: ClickHouse не ответил за ${WAIT_CLICKHOUSE_SECONDS} сек." >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
echo "=== Чистый прогон стартовой истории как источника аналитики ==="
|
||||
echo "GEN_SEED=${GEN_SEED}"
|
||||
echo "GEN_MODEL_T0=${GEN_MODEL_T0}"
|
||||
echo "GEN_MODEL_T_END=${GEN_MODEL_T_END}"
|
||||
echo "GEN_LAMBDA_BASE_PER_MIN=${GEN_LAMBDA_BASE_PER_MIN}"
|
||||
echo ""
|
||||
|
||||
if [[ "${CLEAN_START}" == "1" ]]; then
|
||||
echo "Шаг 0: очистка volumes ClickHouse/Kafka/state"
|
||||
${COMPOSE_BIN} down -v --remove-orphans
|
||||
else
|
||||
echo "Шаг 0: CLEAN_START=0, очистка пропущена"
|
||||
fi
|
||||
|
||||
echo "Шаг 1: запуск ClickHouse и Kafka"
|
||||
${COMPOSE_BIN} up -d clickhouse kafka
|
||||
wait_for_clickhouse
|
||||
|
||||
echo "Шаг 2: применение DDL"
|
||||
bash "${SCRIPT_DIR}/apply_clickhouse_ddl.sh"
|
||||
|
||||
echo "Шаг 3: сборка образа генератора"
|
||||
${COMPOSE_BIN} build generator
|
||||
|
||||
echo "Шаг 4: backfill стартовой истории в Kafka"
|
||||
${COMPOSE_BIN} run --rm --no-deps \
|
||||
-e GEN_RUN_MODE=backfill \
|
||||
-e GEN_STATE_RESET=true \
|
||||
-e GEN_SEED="${GEN_SEED}" \
|
||||
-e GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
-e GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||||
-e GEN_MODEL_TIMEZONE="${GEN_MODEL_TIMEZONE}" \
|
||||
-e GEN_MODEL_TIME_SPEED="${GEN_MODEL_TIME_SPEED}" \
|
||||
-e GEN_TICK_SECONDS="${GEN_TICK_SECONDS}" \
|
||||
-e GEN_LAMBDA_BASE_PER_MIN="${GEN_LAMBDA_BASE_PER_MIN}" \
|
||||
-e GEN_JITTER_PCT="${GEN_JITTER_PCT}" \
|
||||
-e GEN_MIN_EVENTS_PER_TICK="${GEN_MIN_EVENTS_PER_TICK}" \
|
||||
-e GEN_MAX_EVENTS_PER_TICK="${GEN_MAX_EVENTS_PER_TICK}" \
|
||||
generator
|
||||
|
||||
echo "Шаг 5: ожидание чтения Kafka Materialized View (${WAIT_STG_SECONDS} сек.)"
|
||||
sleep "${WAIT_STG_SECONDS}"
|
||||
|
||||
echo "Шаг 6: batch STG -> ODS -> DDS -> DM"
|
||||
bash "${SCRIPT_DIR}/run_batch.sh"
|
||||
|
||||
echo "Шаг 7: инициализация Superset metadata и dashboard"
|
||||
${COMPOSE_BIN} up -d postgres-metadata clickhouse
|
||||
${COMPOSE_BIN} up --abort-on-container-exit --exit-code-from superset-init superset-init
|
||||
${COMPOSE_BIN} up -d --no-deps superset
|
||||
|
||||
echo "Шаг 8: техническая проверка аналитического контура"
|
||||
GEN_MODEL_T0="${GEN_MODEL_T0}" \
|
||||
GEN_MODEL_T_END="${GEN_MODEL_T_END}" \
|
||||
COMPOSE_BIN="${COMPOSE_BIN}" \
|
||||
bash "${SCRIPT_DIR}/check_generated_analytics.sh"
|
||||
|
||||
echo ""
|
||||
echo "Готово: стартовая история генератора доведена до DM и Superset."
|
||||
Reference in New Issue
Block a user