diff --git a/airflow-docker/README.md b/airflow-docker/README.md index 5222126..9c0084f 100644 --- a/airflow-docker/README.md +++ b/airflow-docker/README.md @@ -4,7 +4,8 @@ ## 🎯 Цель проекта -Создать максимально простую среду для изучения Apache Airflow, где все переменные окружения захардкожены для удобства студентов. +Создать максимально простую среду для изучения Apache Airflow. +Для удобства все переменные окружения захардкожены в docker скрипты. На проде так делать не надо :) ## 📋 Предварительные требования @@ -70,7 +71,7 @@ graph TB ``` airflow-docker/ ├── docker-compose.yml # Конфигурация Docker -├── .env # Переменные окружения (создается автоматически) +├── .env # Файл не используется: переменные заданы в docker-compose.yml ├── dags/ # DAG файлы для обучения │ ├── hello_world_dag.py # Базовый пример │ ├── sql_basic_dag.py # Работа с SQL @@ -124,37 +125,23 @@ airflow-docker/ ### Переменные окружения -Все переменные захардкожены для простоты: +В этом проекте мы не используем `.env`: все значения заданы напрямую в `docker-compose.yml`. -```env -# Airflow Configuration -AIRFLOW_USER=admin -AIRFLOW_PASSWORD=admin +Сервисы Airflow получают: +- `POSTGRES_TRAINING_HOST=postgres-training` +- `POSTGRES_TRAINING_PORT=5432` +- `POSTGRES_TRAINING_DB=training` +- `POSTGRES_TRAINING_USER=student` +- `POSTGRES_TRAINING_PASSWORD=student` +- `AIRFLOW_CONN_POSTGRES_TRAINING=postgresql://student:student@postgres-training:5432/training` -# PostgreSQL (Airflow metadata) -PG_USER=airflow -PG_PASSWORD=airflow -PG_DB=airflow +`docker-compose run --rm airflow-init` запускает `airflow connections add postgres_training`, поэтому соединение доступно сразу после инициализации. Проверить наличие можно через: -# PostgreSQL для training exercises -TRAINING_PG_USER=student -TRAINING_PG_PASSWORD=student -TRAINING_PG_DB=training -TRAINING_PG_HOST=postgres-training -TRAINING_PG_PORT=5432 - -# Connection ID for PostgreSQL training database -POSTGRES_CONN_ID=postgres_training - -# CSV Pipeline -CSV_DIR=/opt/airflow/data -CSV_ROWS=1000 - -# Airflow Configuration -_AIRFLOW_DB_UPGRADE=true -_AIRFLOW_WWW_USER_CREATE=true +```bash +docker-compose exec airflow-webserver airflow connections get postgres_training ``` + ### Порты - `8080` - Airflow Webserver diff --git a/airflow-docker/dag-specifications.md b/airflow-docker/dag-specifications.md index 0f39b10..abb3342 100644 --- a/airflow-docker/dag-specifications.md +++ b/airflow-docker/dag-specifications.md @@ -42,6 +42,8 @@ start_task → date_task → end_task - Execute SQL queries - Use PostgresOperator +**Connection Hint:** `docker-compose run --rm airflow-init` automatically provisions the `postgres_training` connection via `airflow connections add`, so no manual setup is required. You can verify it with `docker-compose exec airflow-webserver airflow connections get postgres_training`. + **Tasks:** - `create_table`: Create simple table (users, products) - `insert_data`: Insert sample records @@ -242,4 +244,4 @@ CREATE TABLE enrollments ( - Uses conditional branching - Implements proper error handling - Creates reusable components -- Optimizes workflow performance \ No newline at end of file +- Optimizes workflow performance diff --git a/airflow-docker/docker-compose.yml b/airflow-docker/docker-compose.yml index 7f88de6..b41c79c 100644 --- a/airflow-docker/docker-compose.yml +++ b/airflow-docker/docker-compose.yml @@ -1,3 +1,15 @@ +x-airflow-env: &airflow-default-env + AIRFLOW__CORE__LOAD_EXAMPLES: "False" + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow + AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string + POSTGRES_TRAINING_HOST: postgres-training + POSTGRES_TRAINING_PORT: "5432" + POSTGRES_TRAINING_DB: training + POSTGRES_TRAINING_USER: student + POSTGRES_TRAINING_PASSWORD: student + AIRFLOW_CONN_POSTGRES_TRAINING: postgresql://student:student@postgres-training:5432/training + services: # PostgreSQL for Airflow metadata postgres-metadata: @@ -39,10 +51,7 @@ services: build: . image: airflow-optimized:2.9.2 environment: - AIRFLOW__CORE__LOAD_EXAMPLES: "False" - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow - AIRFLOW__CORE__EXECUTOR: LocalExecutor - AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string + <<: *airflow-default-env command: > bash -c " airflow webserver @@ -65,10 +74,7 @@ services: build: . # image: airflow-optimized:2.9.2 environment: - AIRFLOW__CORE__LOAD_EXAMPLES: "False" - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow - AIRFLOW__CORE__EXECUTOR: LocalExecutor - AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string + <<: *airflow-default-env command: > bash -c " airflow scheduler @@ -90,9 +96,7 @@ services: user: "0:0" # image: airflow-optimized:2.9.2 environment: - AIRFLOW__CORE__LOAD_EXAMPLES: "False" - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow - AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string + <<: *airflow-default-env volumes: - ./dags:/opt/airflow/dags - ./data:/opt/airflow/data @@ -103,7 +107,9 @@ services: chown -R airflow:0 /opt/airflow/data || true && umask 000 && su -s /bin/bash airflow -c 'airflow db migrate' && - su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' + su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' && + su -s /bin/bash airflow -c 'airflow connections delete postgres_training || true' && + su -s /bin/bash airflow -c \"airflow connections add 'postgres_training' --conn-uri \\\"$${AIRFLOW_CONN_POSTGRES_TRAINING}\\\" --conn-description 'Training exercises database'\" " depends_on: postgres-metadata: