Установка зависимостей вынесена в отдельный контейнер

airflow-docker/CHANGELOG.md

@@ -1,70 +0,0 @@
-# CHANGELOG - Educational Airflow Setup
-
-## 📋 Summary of Changes Made
-
-### 1. Fixed Docker Compose Structure
-- Removed duplicate `services:` sections
-- Added missing service definitions (postgres-metadata, postgres-training, airflow-webserver, airflow-scheduler, airflow-init)
-- Configured proper service dependencies and health checks
-
-### 2. Created Hardcoded Environment Variables
-- `.env` file with all necessary credentials:
-  - Airflow: admin/admin
-  - PostgreSQL metadata: airflow/airflow
-  - PostgreSQL training: student/student
-
-### 3. Implemented Educational DAG Examples
-
-**Level 1: Basic Concepts**
-- `hello_world_dag.py` - Simple PythonOperator tasks with dependencies
-
-**Level 2: Intermediate Integration**
-- `sql_basic_dag.py` - PostgreSQL operations
-- `file_operations_dag.py` - File processing and CSV operations
-
-**Level 3: Advanced Features**
-- `data_processing_dag.py` - ETL pipeline with multiple steps
-
-**Level 4: Conditional Logic**
-- `branching_dag.py` - BranchPythonOperator and conditional execution
-
-**Level 5: Error Handling**
-- `error_handling_dag.py` - Task retries and error management
-
-### 4. Sample Data Files
-- `customers.csv` - Customer data for exercises
-- `orders.csv` - Order data for practical work
-
-### 5. Database Configuration
-- PostgreSQL for Airflow metadata (port 5433)
-- PostgreSQL for training exercises (port 5432)
-
-### 6. Configuration Files
-- `requirements.txt` - Essential Python packages
-- Updated `README.md` with comprehensive setup instructions
-
-## 🎯 Key Features for Beginners
-
-### Simplified Setup
-- All environment variables hardcoded in `.env`
-- Clear port mappings and access credentials
-
-### Progressive Learning
-- From basic "Hello World" to advanced ETL pipelines
-- Hands-on exercises with real data
-
-### Technical Specifications
-- Airflow version: 2.9.2
-- PostgreSQL version: 16
-- LocalExecutor for simplicity
-
-## 🚀 Quick Start Commands
-
-```bash
-# Start all services
-docker-compose up -d
-
-# Access interfaces
-- Airflow UI: http://localhost:8080
-- Training PostgreSQL: localhost:5432
-- Metadata PostgreSQL: localhost:5433

airflow-docker/Dockerfile

@@ -0,0 +1,26 @@
+# Use the official Airflow image as base
+FROM apache/airflow:2.9.2
+
+# Set environment variables
+ENV AIRFLOW_HOME=/opt/airflow
+
+# Copy the optimized requirements file
+COPY requirements-optimized.txt .
+
+# Install the required Python packages
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Create necessary directories
+RUN mkdir -p /opt/airflow/dags /opt/airflow/logs /opt/airflow/config /opt/airflow/data
+
+# Set working directory
+WORKDIR ${AIRFLOW_HOME}
+
+# Change ownership to airflow user
+RUN chown -R airflow:airflow ${AIRFLOW_HOME}
+
+# Expose the webserver port
+EXPOSE 8080
+
+# The image will use Airflow's default entrypoint
+# Commands will be passed at runtime

airflow-docker/docker-compose.yml

@@ -38,14 +38,14 @@ services:
 
   # Airflow webserver
   airflow-webserver:
-    image: apache/airflow:2.9.2
+    build: .
+    image: airflow-optimized:2.9.2
     environment:
       AIRFLOW__CORE__LOAD_EXAMPLES: "False"
-      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
+      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
       AIRFLOW__CORE__EXECUTOR: LocalExecutor
     command: >
       bash -c "
-        pip install --no-cache-dir psycopg2-binary &&
         airflow webserver
       "
     ports:
@@ -61,14 +61,14 @@ services:
 
   # Airflow scheduler
   airflow-scheduler:
-    image: apache/airflow:2.9.2
+    build: .
+    # image: airflow-optimized:2.9.2
     environment:
       AIRFLOW__CORE__LOAD_EXAMPLES: "False"
-      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
+      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
       AIRFLOW__CORE__EXECUTOR: LocalExecutor
     command: >
       bash -c "
-        pip install --no-cache-dir psycopg2-binary &&
         airflow scheduler
       "
     volumes:
@@ -82,16 +82,16 @@ services:
 
   # Airflow init to create admin user
   airflow-init:
-    image: apache/airflow:2.9.2
+    build: .
+    # image: airflow-optimized:2.9.2
     environment:
       AIRFLOW__CORE__LOAD_EXAMPLES: "False"
-      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
+      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
     volumes:
       - ./dags:/opt/airflow/dags
       - ./data:/opt/airflow/data
     command: >
       bash -c "
-        pip install --no-cache-dir psycopg2-binary &&
         airflow db migrate &&
         airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org
       "

airflow-docker/requirements.txt

@@ -1,13 +1,11 @@
-# Core packages for Airflow
+# Optimized requirements for Airflow Docker setup
+# Only includes packages actually used in DAGs
+
+# Core database connector
 psycopg2-binary==2.9.7
+
+# Data processing (used in data_processing_dag.py and file_operations_dag.py)
 pandas==2.1.4
-numpy==1.24.3
 
-# Additional packages for data processing
-openpyxl==3.1.2
-xlrd==2.0.1
-python-dateutil==2.8.2
-
-# For additional operators (if needed)
-apache-airflow-providers-postgres==8.5.1
-apache-airflow-providers-common-sql==1.13.0
+# Airflow PostgreSQL provider (used in sql_basic_dag.py and data_processing_dag.py)
+apache-airflow-providers-postgres==8.5.1