diff --git a/airflow-docker/CHANGELOG.md b/airflow-docker/CHANGELOG.md deleted file mode 100644 index 4ed9e1c..0000000 --- a/airflow-docker/CHANGELOG.md +++ /dev/null @@ -1,70 +0,0 @@ -# CHANGELOG - Educational Airflow Setup - -## 📋 Summary of Changes Made - -### 1. Fixed Docker Compose Structure -- Removed duplicate `services:` sections -- Added missing service definitions (postgres-metadata, postgres-training, airflow-webserver, airflow-scheduler, airflow-init) -- Configured proper service dependencies and health checks - -### 2. Created Hardcoded Environment Variables -- `.env` file with all necessary credentials: - - Airflow: admin/admin - - PostgreSQL metadata: airflow/airflow - - PostgreSQL training: student/student - -### 3. Implemented Educational DAG Examples - -**Level 1: Basic Concepts** -- `hello_world_dag.py` - Simple PythonOperator tasks with dependencies - -**Level 2: Intermediate Integration** -- `sql_basic_dag.py` - PostgreSQL operations -- `file_operations_dag.py` - File processing and CSV operations - -**Level 3: Advanced Features** -- `data_processing_dag.py` - ETL pipeline with multiple steps - -**Level 4: Conditional Logic** -- `branching_dag.py` - BranchPythonOperator and conditional execution - -**Level 5: Error Handling** -- `error_handling_dag.py` - Task retries and error management - -### 4. Sample Data Files -- `customers.csv` - Customer data for exercises -- `orders.csv` - Order data for practical work - -### 5. Database Configuration -- PostgreSQL for Airflow metadata (port 5433) -- PostgreSQL for training exercises (port 5432) - -### 6. Configuration Files -- `requirements.txt` - Essential Python packages -- Updated `README.md` with comprehensive setup instructions - -## 🎯 Key Features for Beginners - -### Simplified Setup -- All environment variables hardcoded in `.env` -- Clear port mappings and access credentials - -### Progressive Learning -- From basic "Hello World" to advanced ETL pipelines -- Hands-on exercises with real data - -### Technical Specifications -- Airflow version: 2.9.2 -- PostgreSQL version: 16 -- LocalExecutor for simplicity - -## 🚀 Quick Start Commands - -```bash -# Start all services -docker-compose up -d - -# Access interfaces -- Airflow UI: http://localhost:8080 -- Training PostgreSQL: localhost:5432 -- Metadata PostgreSQL: localhost:5433 \ No newline at end of file diff --git a/airflow-docker/Dockerfile b/airflow-docker/Dockerfile new file mode 100644 index 0000000..336fa21 --- /dev/null +++ b/airflow-docker/Dockerfile @@ -0,0 +1,26 @@ +# Use the official Airflow image as base +FROM apache/airflow:2.9.2 + +# Set environment variables +ENV AIRFLOW_HOME=/opt/airflow + +# Copy the optimized requirements file +COPY requirements-optimized.txt . + +# Install the required Python packages +RUN pip install --no-cache-dir -r requirements.txt + +# Create necessary directories +RUN mkdir -p /opt/airflow/dags /opt/airflow/logs /opt/airflow/config /opt/airflow/data + +# Set working directory +WORKDIR ${AIRFLOW_HOME} + +# Change ownership to airflow user +RUN chown -R airflow:airflow ${AIRFLOW_HOME} + +# Expose the webserver port +EXPOSE 8080 + +# The image will use Airflow's default entrypoint +# Commands will be passed at runtime \ No newline at end of file diff --git a/airflow-docker/docker-compose.yml b/airflow-docker/docker-compose.yml index d74328b..9da968e 100644 --- a/airflow-docker/docker-compose.yml +++ b/airflow-docker/docker-compose.yml @@ -38,14 +38,14 @@ services: # Airflow webserver airflow-webserver: - image: apache/airflow:2.9.2 + build: . + image: airflow-optimized:2.9.2 environment: AIRFLOW__CORE__LOAD_EXAMPLES: "False" - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow AIRFLOW__CORE__EXECUTOR: LocalExecutor command: > bash -c " - pip install --no-cache-dir psycopg2-binary && airflow webserver " ports: @@ -61,14 +61,14 @@ services: # Airflow scheduler airflow-scheduler: - image: apache/airflow:2.9.2 + build: . + # image: airflow-optimized:2.9.2 environment: AIRFLOW__CORE__LOAD_EXAMPLES: "False" - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow AIRFLOW__CORE__EXECUTOR: LocalExecutor command: > bash -c " - pip install --no-cache-dir psycopg2-binary && airflow scheduler " volumes: @@ -82,16 +82,16 @@ services: # Airflow init to create admin user airflow-init: - image: apache/airflow:2.9.2 + build: . + # image: airflow-optimized:2.9.2 environment: AIRFLOW__CORE__LOAD_EXAMPLES: "False" - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow volumes: - ./dags:/opt/airflow/dags - ./data:/opt/airflow/data command: > bash -c " - pip install --no-cache-dir psycopg2-binary && airflow db migrate && airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org " diff --git a/airflow-docker/requirements.txt b/airflow-docker/requirements.txt index 1a31267..05cd212 100644 --- a/airflow-docker/requirements.txt +++ b/airflow-docker/requirements.txt @@ -1,13 +1,11 @@ -# Core packages for Airflow +# Optimized requirements for Airflow Docker setup +# Only includes packages actually used in DAGs + +# Core database connector psycopg2-binary==2.9.7 + +# Data processing (used in data_processing_dag.py and file_operations_dag.py) pandas==2.1.4 -numpy==1.24.3 -# Additional packages for data processing -openpyxl==3.1.2 -xlrd==2.0.1 -python-dateutil==2.8.2 - -# For additional operators (if needed) -apache-airflow-providers-postgres==8.5.1 -apache-airflow-providers-common-sql==1.13.0 \ No newline at end of file +# Airflow PostgreSQL provider (used in sql_basic_dag.py and data_processing_dag.py) +apache-airflow-providers-postgres==8.5.1 \ No newline at end of file