From 0d81096dedc57de2bb2ede0f1ed07f33db49bfc2 Mon Sep 17 00:00:00 2001 From: Dmitry Dementev Date: Thu, 11 Dec 2025 10:51:39 +0300 Subject: [PATCH] =?UTF-8?q?=D0=BF=D0=BE=D0=BB=D0=BD=D1=8B=D0=B9=20smoke=20?= =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=20=D1=81=D1=82=D0=B5=D0=BD=D0=B4=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 4 ++++ README.md | 1 + scripts/e2e_smoke.sh | 53 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100755 scripts/e2e_smoke.sh diff --git a/Makefile b/Makefile index 2ddabcb..4ea395d 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ BOOKINGS_INIT_DAYS ?= 1 .PHONY: up stop down clean airflow-init logs gp-psql ddl-gp \ bookings-clone-demodb bookings-init bookings-psql bookings-generate-day \ dev-setup dev-sync dev-lock test lint fmt clean-venv +SHELL := /bin/bash up: docker compose -f docker-compose.yml up -d @@ -113,3 +114,6 @@ fmt: clean-venv: python -c "import shutil; shutil.rmtree('.venv', ignore_errors=True)" + +e2e-smoke: + ./scripts/e2e_smoke.sh diff --git a/README.md b/README.md index 97057f0..f6ece16 100644 --- a/README.md +++ b/README.md @@ -421,5 +421,6 @@ load_bookings_to_stg = PostgresOperator( - Локальные проверки: `make test` (pytest). Для форматирования — `make fmt`, для проверки — `make lint`. - Пошаговый сценарий с Docker (включая негативные кейсы и reset) — см. `TESTING.md`. +- Полный smoke-тест стенда (сносит volumes!): `make e2e-smoke` — поднимает стек с нуля и прогоняет `csv_to_greenplum` и `bookings_to_gp_stage` через `airflow dags test`. Удачи в изучении Data Engineering! 🚀 diff --git a/scripts/e2e_smoke.sh b/scripts/e2e_smoke.sh new file mode 100755 index 0000000..4846a85 --- /dev/null +++ b/scripts/e2e_smoke.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Полный smoke-тест стенда: сносит volumes, поднимает стек, +# и проверяет оба учебных DAG через airflow dags test. + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +warn() { echo ":: ${1}" >&2; } + +wait_up() { + local svc="$1" + local attempts=60 + while true; do + if docker compose -f docker-compose.yml ps "$svc" 2>/dev/null | grep -q "Up"; then + break + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Service $svc is not up after waiting" >&2 + exit 1 + fi + sleep 2 + done +} + +warn "Reset stack (containers + volumes)" +make clean + +warn "Starting stack (docker compose up)" +make up + +warn "Waiting for Airflow services" +wait_up airflow-webserver +wait_up airflow-scheduler + +warn "Init demo DB bookings" +make bookings-init + +warn "Apply DDL to Greenplum" +make ddl-gp + +warn "Run local pytest suite" +make test + +warn "Airflow DAG test: csv_to_greenplum" +docker compose -f docker-compose.yml exec airflow-webserver airflow dags test csv_to_greenplum 2024-01-01 + +warn "Airflow DAG test: bookings_to_gp_stage" +docker compose -f docker-compose.yml exec airflow-webserver airflow dags test bookings_to_gp_stage 2024-01-01 + +warn "Smoke test completed successfully"