feat(infra): add airflow orchestration services

Add Apache Airflow infrastructure with webserver, scheduler, and metadata
database to enable DAG-based pipeline orchestration. Includes optimized
requirements file and Docker configuration for Airflow 2.9.3.

Dockerfile.airflow

@@ -0,0 +1,23 @@
+# Use the official Airflow image as base
+FROM apache/airflow:2.9.3
+
+# Set environment variables
+ENV AIRFLOW_HOME=/opt/airflow
+
+# Copy the project requirements file
+COPY --chown=airflow:0 requirements.txt ${AIRFLOW_HOME}/requirements.txt
+
+# Install the required Python packages
+RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt
+
+# Create necessary directories
+RUN mkdir -p /opt/airflow/dags /opt/airflow/logs /opt/airflow/config /opt/airflow/data
+
+# Set working directory
+WORKDIR ${AIRFLOW_HOME}
+
+# Expose the webserver port
+EXPOSE 8080
+
+# The image will use Airflow's default entrypoint
+# Commands will be passed at runtime

airflow/requirements.txt

@@ -0,0 +1,14 @@
+# Optimized requirements for Airflow Docker setup
+# Only includes packages actually used in DAGs
+
+# Core database connector
+psycopg2-binary==2.9.9
+
+# Data processing (used in data_processing_dag.py and file_operations_dag.py)
+pandas==2.1.4
+
+# Data generation library (used in file_operations_dag.py)
+mimesis==15.1.0
+
+# Airflow PostgreSQL provider (used in sql_basic_dag.py and data_processing_dag.py)
+apache-airflow-providers-postgres==5.11.1

docker-compose.yml

@@ -1,3 +1,16 @@
+x-airflow-env: &airflow-default-env
+  AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
+  AIRFLOW__CORE__EXECUTOR: LocalExecutor
+  AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string
+  POSTGRES_TRAINING_HOST: postgres-training
+  POSTGRES_TRAINING_PORT: "5432"
+  POSTGRES_TRAINING_DB: training
+  POSTGRES_TRAINING_USER: student
+  POSTGRES_TRAINING_PASSWORD: student
+  AIRFLOW_CONN_POSTGRES_TRAINING: postgresql://student:student@postgres-training:5432/training
+
+
 services:
   clickhouse:
     image: clickhouse/clickhouse-server:25.1
@@ -64,6 +77,100 @@ services:
     networks:
       - cs_dwh
 
+  # PostgreSQL for Airflow metadata
+  postgres-metadata:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: airflow
+      POSTGRES_PASSWORD: airflow
+      POSTGRES_DB: airflow
+    ports:
+      - "5434:5432"
+    volumes:
+      - pgmeta:/var/lib/postgresql/data
+    networks:
+      - cs_dwh
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U airflow -d airflow"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+
+  # Airflow webserver
+  airflow-webserver:
+    build: .
+    image: airflow-optimized:2.9.2
+    environment:
+      <<: *airflow-default-env
+    command: >
+      bash -c "
+        airflow webserver
+      "
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./dags:/opt/airflow/dags
+      - ./data:/opt/airflow/data
+    networks:
+      - cs_dwh
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully
+      postgres-metadata:
+        condition: service_healthy
+      postgres-training:
+        condition: service_healthy
+
+  # Airflow scheduler
+  airflow-scheduler:
+    build: .
+    # image: airflow-optimized:2.9.2
+    environment:
+      <<: *airflow-default-env
+    command: >
+      bash -c "
+        airflow scheduler
+      "
+    volumes:
+      - ./dags:/opt/airflow/dags
+      - ./data:/opt/airflow/data
+    networks:
+      - cs_dwh
+    depends_on:
+      airflow-init:
+        condition: service_completed_successfully
+      postgres-metadata:
+        condition: service_healthy
+      postgres-training:
+        condition: service_healthy
+
+  # Airflow init to create admin user
+  airflow-init:
+    build: .
+    user: "0:0"
+    # image: airflow-optimized:2.9.3
+    environment:
+      <<: *airflow-default-env
+    volumes:
+      - ./dags:/opt/airflow/dags
+      - ./data:/opt/airflow/data
+    networks:
+      - cs_dwh
+    command: >
+      bash -ceuo pipefail "
+        mkdir -p /opt/airflow/data &&
+        chmod -R 777 /opt/airflow/data || true &&
+        chown -R airflow:0 /opt/airflow/data || true &&
+        umask 000 &&
+        su -s /bin/bash airflow -c 'airflow db migrate' &&
+        su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' &&
+        su -s /bin/bash airflow -c 'airflow connections delete postgres_training || true' &&
+        su -s /bin/bash airflow -c \"airflow connections add 'postgres_training' --conn-uri \\\"$${AIRFLOW_CONN_POSTGRES_TRAINING}\\\" --conn-description 'Training exercises database'\"
+      "
+    depends_on:
+      postgres-metadata:
+        condition: service_healthy
+
   prometheus:
     image: prom/prometheus:v2.53.4
     volumes:
@@ -127,6 +234,7 @@ networks:
 volumes:
   grafana_lib:
   kafka-data:
+  pgmeta:
   superset_data:
   superset_config: