ddadmin
af6b0a9fae
- Зачем: - сделать примеры более доступными для начинающих, убрав лишнюю архитектурную сложность (хелперы, внешние SQL файлы). - сфокусировать обучение на самом Airflow, а не на структуре Python-проекта. - Что: - консолидирована логика: DDL и вспомогательные функции перенесены из helpers/ и sql/ напрямую в csv_to_postgres.py и csv_to_postgres_dq.py. - удалена избыточная инфраструктура: папки sql/, tests/, helpers/ и файл requirements.txt больше не содержат специфичных для этого пайплайна файлов. - упрощена работа с БД: использование PostgresHook напрямую в задачах вместо кастомных оберток. - исправлен порт PostgreSQL для метаданных Airflow (5433 → 5434) в README.md и educational-setup-plan.md. - изменен путь генерации CSV на /opt/airflow/data/output в csv_to_postgres.py. - добавлен шаблон data/*.csv в .gitignore. - обновлен комментарий в sql_basic_dag.py о создании соединения. - исправлено описание практического задания в educational-tasks.md. - добавлены переводы строк в конце файлов csv_to_postgres.py и educational-setup-plan.md. - Проверка: - запуск DAG-ов csv_to_postgres и csv_to_postgres_dq в Airflow UI.
121 lines
3.5 KiB
YAML
121 lines
3.5 KiB
YAML
x-airflow-env: &airflow-default-env
|
|
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
|
|
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
|
AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string
|
|
POSTGRES_TRAINING_HOST: postgres-training
|
|
POSTGRES_TRAINING_PORT: "5432"
|
|
POSTGRES_TRAINING_DB: training
|
|
POSTGRES_TRAINING_USER: student
|
|
POSTGRES_TRAINING_PASSWORD: student
|
|
AIRFLOW_CONN_POSTGRES_TRAINING: postgresql://student:student@postgres-training:5432/training
|
|
|
|
services:
|
|
# PostgreSQL for Airflow metadata
|
|
postgres-metadata:
|
|
image: postgres:16
|
|
environment:
|
|
POSTGRES_USER: airflow
|
|
POSTGRES_PASSWORD: airflow
|
|
POSTGRES_DB: airflow
|
|
ports:
|
|
- "5434:5432"
|
|
volumes:
|
|
- pgmeta:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U airflow -d airflow"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 20
|
|
|
|
# PostgreSQL for training exercises
|
|
postgres-training:
|
|
image: postgres:16
|
|
environment:
|
|
POSTGRES_USER: student
|
|
POSTGRES_PASSWORD: student
|
|
POSTGRES_DB: training
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- pg_data:/var/lib/postgresql/data
|
|
- ./init-postgres.sql:/docker-entrypoint-initdb.d/init-postgres.sql
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U student -d training"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 20
|
|
|
|
# Airflow webserver
|
|
airflow-webserver:
|
|
build: .
|
|
image: airflow-optimized:2.9.2
|
|
environment:
|
|
<<: *airflow-default-env
|
|
command: >
|
|
bash -c "
|
|
airflow webserver
|
|
"
|
|
ports:
|
|
- "8080:8080"
|
|
volumes:
|
|
- ./dags:/opt/airflow/dags
|
|
- ./data:/opt/airflow/data
|
|
depends_on:
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
postgres-metadata:
|
|
condition: service_healthy
|
|
postgres-training:
|
|
condition: service_healthy
|
|
|
|
# Airflow scheduler
|
|
airflow-scheduler:
|
|
build: .
|
|
# image: airflow-optimized:2.9.2
|
|
environment:
|
|
<<: *airflow-default-env
|
|
command: >
|
|
bash -c "
|
|
airflow scheduler
|
|
"
|
|
volumes:
|
|
- ./dags:/opt/airflow/dags
|
|
- ./data:/opt/airflow/data
|
|
depends_on:
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
postgres-metadata:
|
|
condition: service_healthy
|
|
postgres-training:
|
|
condition: service_healthy
|
|
|
|
# Airflow init to create admin user
|
|
airflow-init:
|
|
build: .
|
|
user: "0:0"
|
|
# image: airflow-optimized:2.9.2
|
|
environment:
|
|
<<: *airflow-default-env
|
|
volumes:
|
|
- ./dags:/opt/airflow/dags
|
|
- ./data:/opt/airflow/data
|
|
command: >
|
|
bash -ceuo pipefail "
|
|
mkdir -p /opt/airflow/data &&
|
|
chmod -R 777 /opt/airflow/data || true &&
|
|
chown -R airflow:0 /opt/airflow/data || true &&
|
|
umask 000 &&
|
|
su -s /bin/bash airflow -c 'airflow db migrate' &&
|
|
su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' &&
|
|
su -s /bin/bash airflow -c 'airflow connections delete postgres_training || true' &&
|
|
su -s /bin/bash airflow -c \"airflow connections add 'postgres_training' --conn-uri \\\"$${AIRFLOW_CONN_POSTGRES_TRAINING}\\\" --conn-description 'Training exercises database'\"
|
|
"
|
|
depends_on:
|
|
postgres-metadata:
|
|
condition: service_healthy
|
|
|
|
volumes:
|
|
pgmeta:
|
|
pg_data:
|