fix(bookings): исправлена генерация demodb и добавлен seed-дамп
- Зачем:
- генерация с нуля занимала часы, boarding_passes были пустыми, данные
пропадали после docker compose down/up.
- Что:
- обновлён demodb до коммита 866e56f, добавлен патч install_connstr_no_hardcode.
- BOOKINGS_INIT_DAYS увеличен до 60, BOOKINGS_JOBS по умолчанию 2.
- добавлен seed-дамп bookings/seed/demo.sql.xz (42 MB, xz вместо 7z).
- make bookings-init восстанавливает из дампа (~18 сек) и применяет GUC из .env.
- make bookings-generate — генерация с нуля для разработчиков.
- generate_next_day.sql: COMMIT после continue(), pg_sleep(3) для jobs>1, synchronous_commit=on + CHECKPOINT.
- bookings-check-jobs: добавлена валидация нечисловых значений BOOKINGS_JOBS.
- bookings-init теперь вызывает bookings-check-jobs как prerequisite.
- синхронизированы внутренние документы (коммит demodb, init_days=60).
- Проверка:
- make bookings-init && make bookings-generate-day BOOKINGS_JOBS=2.
- make test.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -2,14 +2,16 @@ SHELL := /bin/bash
|
||||
UV := uv
|
||||
PYTHON_VERSION := 3.11
|
||||
DEMODB_REPO := https://github.com/postgrespro/demodb.git
|
||||
DEMODB_COMMIT := d68de192850237719f09b47688d5f3fc94653ca6
|
||||
BOOKINGS_JOBS ?= 1
|
||||
DEMODB_COMMIT := 866e56f7fe54596a1d2a88f5f32f4aa3b2698121
|
||||
BOOKINGS_JOBS ?= 2
|
||||
BOOKINGS_START_DATE ?= 2017-01-01
|
||||
BOOKINGS_INIT_DAYS ?= 1
|
||||
BOOKINGS_INIT_DAYS ?= 60
|
||||
|
||||
.PHONY: up stop down clean airflow-init logs gp-psql ddl-gp \
|
||||
bookings-check-jobs bookings-clone-demodb bookings-init bookings-psql bookings-generate-day \
|
||||
bookings-check-jobs bookings-clone-demodb bookings-init bookings-generate bookings-dump \
|
||||
bookings-psql bookings-generate-day \
|
||||
dev-setup dev-sync dev-lock test lint fmt clean-venv build e2e-smoke e2e-etl
|
||||
BOOKINGS_SEED := bookings/seed/demo.sql.xz
|
||||
SHELL := /bin/bash
|
||||
|
||||
up:
|
||||
@@ -52,25 +54,35 @@ bookings-clone-demodb:
|
||||
# Патчим generate/continue: при jobs=1 запускаем process_queue синхронно, без dblink
|
||||
if ! grep -q "Job 1 (local): ok" bookings/demodb/engine.sql; then \
|
||||
if ! patch -d bookings/demodb -p1 --forward < bookings/patches/engine_jobs1_sync.patch; then \
|
||||
echo "Не удалось применить патч engine_jobs1_sync.patch. Удалите bookings/demodb и повторите make bookings-init." >&2; \
|
||||
echo "Не удалось применить патч engine_jobs1_sync.patch. Удалите bookings/demodb и повторите make bookings-generate." >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
fi
|
||||
# Делаем установку идемпотентной и принудительной: DROP DATABASE IF EXISTS demo WITH (FORCE)
|
||||
if ! grep -q "DROP DATABASE IF EXISTS demo WITH (FORCE);" bookings/demodb/install.sql; then \
|
||||
if ! patch -d bookings/demodb -p1 --forward < bookings/patches/install_drop_if_exists.patch; then \
|
||||
echo "Не удалось применить патч install_drop_if_exists.patch. Удалите bookings/demodb и повторите make bookings-init." >&2; \
|
||||
echo "Не удалось применить патч install_drop_if_exists.patch. Удалите bookings/demodb и повторите make bookings-generate." >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
fi
|
||||
# Убираем хардкод gen.connstr без credentials — иначе VACUUM падает
|
||||
if grep -q "gen.connstr = 'dbname=demo'" bookings/demodb/install.sql; then \
|
||||
if ! patch -d bookings/demodb -p1 --forward < bookings/patches/install_connstr_no_hardcode.patch; then \
|
||||
echo "Не удалось применить патч install_connstr_no_hardcode.patch. Удалите bookings/demodb и повторите make bookings-generate." >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
fi
|
||||
|
||||
bookings-check-jobs:
|
||||
@if [ "$(BOOKINGS_JOBS)" != "1" ]; then \
|
||||
echo "Поддерживается только BOOKINGS_JOBS=1. Измените .env и повторите команду." >&2; \
|
||||
@case "$(BOOKINGS_JOBS)" in \
|
||||
''|*[!0-9]*) echo "BOOKINGS_JOBS должен быть целым числом >= 1. Текущее значение: '$(BOOKINGS_JOBS)'." >&2; exit 1;; \
|
||||
esac; \
|
||||
if [ "$(BOOKINGS_JOBS)" -lt 1 ]; then \
|
||||
echo "BOOKINGS_JOBS должен быть >= 1. Текущее значение: $(BOOKINGS_JOBS)." >&2; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
bookings-init: bookings-check-jobs bookings-clone-demodb
|
||||
bookings-generate: bookings-check-jobs bookings-clone-demodb
|
||||
docker compose -f docker-compose.yml up -d bookings-db
|
||||
docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
until PGPASSWORD="$$POSTGRES_PASSWORD" pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" -h localhost; do \
|
||||
@@ -89,10 +101,96 @@ bookings-init: bookings-check-jobs bookings-clone-demodb
|
||||
ALTER DATABASE demo SET bookings.init_days = '\''$(BOOKINGS_INIT_DAYS)'\''; \
|
||||
ALTER DATABASE demo SET bookings.jobs = '\''$(BOOKINGS_JOBS)'\'';" \
|
||||
'
|
||||
# Генерируем первый день данных, чтобы база не оставалась пустой
|
||||
# Генерируем данные за $(BOOKINGS_INIT_DAYS) дней
|
||||
docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" psql -v ON_ERROR_STOP=1 -U "$$POSTGRES_USER" -d demo -f /bookings/generate_next_day.sql \
|
||||
'
|
||||
# Валидация: ключевые таблицы не должны быть пустыми
|
||||
@docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" psql -v ON_ERROR_STOP=1 -U "$$POSTGRES_USER" -d demo -tAc " \
|
||||
SELECT format(E'\''%-25s %s'\'', t, cnt) \
|
||||
FROM ( \
|
||||
SELECT '\''bookings.bookings'\'' AS t, count(*) AS cnt FROM bookings.bookings \
|
||||
UNION ALL \
|
||||
SELECT '\''bookings.tickets'\'', count(*) FROM bookings.tickets \
|
||||
UNION ALL \
|
||||
SELECT '\''bookings.flights'\'', count(*) FROM bookings.flights \
|
||||
UNION ALL \
|
||||
SELECT '\''bookings.boarding_passes'\'', count(*) FROM bookings.boarding_passes \
|
||||
) s ORDER BY t; \
|
||||
" \
|
||||
'
|
||||
@docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" psql -v ON_ERROR_STOP=1 -U "$$POSTGRES_USER" -d demo -tAc " \
|
||||
DO \$$\$$ \
|
||||
DECLARE v_cnt bigint; \
|
||||
BEGIN \
|
||||
SELECT count(*) INTO v_cnt FROM bookings.bookings; \
|
||||
IF v_cnt = 0 THEN \
|
||||
RAISE EXCEPTION '\''bookings.bookings пуста — генерация не сработала. Проверьте логи: SELECT * FROM gen.log ORDER BY at DESC LIMIT 10;'\''; \
|
||||
END IF; \
|
||||
SELECT count(*) INTO v_cnt FROM bookings.flights; \
|
||||
IF v_cnt = 0 THEN \
|
||||
RAISE EXCEPTION '\''bookings.flights пуста — попробуйте увеличить BOOKINGS_INIT_DAYS.'\''; \
|
||||
END IF; \
|
||||
END \$$\$$; \
|
||||
" \
|
||||
'
|
||||
|
||||
bookings-init: bookings-check-jobs
|
||||
@if [ ! -f $(BOOKINGS_SEED) ]; then \
|
||||
echo "Файл $(BOOKINGS_SEED) не найден. Используйте make bookings-generate для генерации с нуля." >&2; \
|
||||
exit 1; \
|
||||
fi
|
||||
docker compose -f docker-compose.yml up -d bookings-db
|
||||
docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
until PGPASSWORD="$$POSTGRES_PASSWORD" pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" -h localhost; do \
|
||||
echo "Waiting for bookings-db to become ready..."; \
|
||||
sleep 1; \
|
||||
done \
|
||||
'
|
||||
docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" psql -v ON_ERROR_STOP=1 -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" \
|
||||
-c "DROP DATABASE IF EXISTS demo WITH (FORCE);" \
|
||||
'
|
||||
@echo "Восстановление из дампа (~42 MB)..."
|
||||
xz -dc $(BOOKINGS_SEED) | docker compose -f docker-compose.yml exec -T bookings-db bash -lc '\
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" psql -v ON_ERROR_STOP=1 -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" \
|
||||
'
|
||||
# Применяем настройки из текущего окружения поверх дампа
|
||||
docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
CONNSTR="dbname=demo user=$$POSTGRES_USER password=$$POSTGRES_PASSWORD"; \
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" psql -v ON_ERROR_STOP=1 -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" -c "\
|
||||
ALTER DATABASE demo SET gen.connstr='\''$$CONNSTR'\''; \
|
||||
ALTER DATABASE demo SET bookings.start_date = '\''$(BOOKINGS_START_DATE)'\''; \
|
||||
ALTER DATABASE demo SET bookings.init_days = '\''$(BOOKINGS_INIT_DAYS)'\''; \
|
||||
ALTER DATABASE demo SET bookings.jobs = '\''$(BOOKINGS_JOBS)'\'';" \
|
||||
'
|
||||
@echo "Восстановление завершено. Проверяем данные..."
|
||||
@docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" psql -U "$$POSTGRES_USER" -d demo -tAc " \
|
||||
SELECT format(E'\''%-25s %s'\'', t, cnt) \
|
||||
FROM ( \
|
||||
SELECT '\''bookings.bookings'\'' AS t, count(*) AS cnt FROM bookings.bookings \
|
||||
UNION ALL \
|
||||
SELECT '\''bookings.tickets'\'', count(*) FROM bookings.tickets \
|
||||
UNION ALL \
|
||||
SELECT '\''bookings.flights'\'', count(*) FROM bookings.flights \
|
||||
UNION ALL \
|
||||
SELECT '\''bookings.boarding_passes'\'', count(*) FROM bookings.boarding_passes \
|
||||
) s ORDER BY t; \
|
||||
" \
|
||||
'
|
||||
|
||||
bookings-dump:
|
||||
@echo "Создание дампа demo → $(BOOKINGS_SEED)..."
|
||||
mkdir -p bookings/seed
|
||||
docker compose -f docker-compose.yml exec bookings-db bash -lc '\
|
||||
PGPASSWORD="$$POSTGRES_PASSWORD" pg_dump -U "$$POSTGRES_USER" -d demo --format=plain --create \
|
||||
' > /tmp/demo_dump_$$$$.sql
|
||||
xz -9 < /tmp/demo_dump_$$$$.sql > $(BOOKINGS_SEED)
|
||||
rm -f /tmp/demo_dump_$$$$.sql
|
||||
@echo "Дамп сохранён: $$(ls -lh $(BOOKINGS_SEED) | awk '{print $$5}')"
|
||||
|
||||
bookings-psql:
|
||||
docker compose -f docker-compose.yml exec bookings-db bash -lc 'PGPASSWORD="$$POSTGRES_PASSWORD" psql -U "$$POSTGRES_USER" -d demo'
|
||||
|
||||
Reference in New Issue
Block a user