diff --git a/Dockerfile.airflow b/Dockerfile.airflow new file mode 100644 index 0000000..0df3b0d --- /dev/null +++ b/Dockerfile.airflow @@ -0,0 +1,23 @@ +# Use the official Airflow image as base +FROM apache/airflow:2.9.3 + +# Set environment variables +ENV AIRFLOW_HOME=/opt/airflow + +# Copy the project requirements file +COPY --chown=airflow:0 requirements.txt ${AIRFLOW_HOME}/requirements.txt + +# Install the required Python packages +RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt + +# Create necessary directories +RUN mkdir -p /opt/airflow/dags /opt/airflow/logs /opt/airflow/config /opt/airflow/data + +# Set working directory +WORKDIR ${AIRFLOW_HOME} + +# Expose the webserver port +EXPOSE 8080 + +# The image will use Airflow's default entrypoint +# Commands will be passed at runtime diff --git a/airflow/requirements.txt b/airflow/requirements.txt new file mode 100644 index 0000000..ba5c858 --- /dev/null +++ b/airflow/requirements.txt @@ -0,0 +1,14 @@ +# Optimized requirements for Airflow Docker setup +# Only includes packages actually used in DAGs + +# Core database connector +psycopg2-binary==2.9.9 + +# Data processing (used in data_processing_dag.py and file_operations_dag.py) +pandas==2.1.4 + +# Data generation library (used in file_operations_dag.py) +mimesis==15.1.0 + +# Airflow PostgreSQL provider (used in sql_basic_dag.py and data_processing_dag.py) +apache-airflow-providers-postgres==5.11.1 diff --git a/docker-compose.yml b/docker-compose.yml index fc6b0a4..4cad047 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,16 @@ +x-airflow-env: &airflow-default-env + AIRFLOW__CORE__LOAD_EXAMPLES: "False" + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow + AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string + POSTGRES_TRAINING_HOST: postgres-training + POSTGRES_TRAINING_PORT: "5432" + POSTGRES_TRAINING_DB: training + POSTGRES_TRAINING_USER: student + POSTGRES_TRAINING_PASSWORD: student + AIRFLOW_CONN_POSTGRES_TRAINING: postgresql://student:student@postgres-training:5432/training + + services: clickhouse: image: clickhouse/clickhouse-server:25.1 @@ -64,6 +77,100 @@ services: networks: - cs_dwh + # PostgreSQL for Airflow metadata + postgres-metadata: + image: postgres:16 + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + ports: + - "5434:5432" + volumes: + - pgmeta:/var/lib/postgresql/data + networks: + - cs_dwh + healthcheck: + test: ["CMD-SHELL", "pg_isready -U airflow -d airflow"] + interval: 5s + timeout: 5s + retries: 20 + + # Airflow webserver + airflow-webserver: + build: . + image: airflow-optimized:2.9.2 + environment: + <<: *airflow-default-env + command: > + bash -c " + airflow webserver + " + ports: + - "8080:8080" + volumes: + - ./dags:/opt/airflow/dags + - ./data:/opt/airflow/data + networks: + - cs_dwh + depends_on: + airflow-init: + condition: service_completed_successfully + postgres-metadata: + condition: service_healthy + postgres-training: + condition: service_healthy + + # Airflow scheduler + airflow-scheduler: + build: . + # image: airflow-optimized:2.9.2 + environment: + <<: *airflow-default-env + command: > + bash -c " + airflow scheduler + " + volumes: + - ./dags:/opt/airflow/dags + - ./data:/opt/airflow/data + networks: + - cs_dwh + depends_on: + airflow-init: + condition: service_completed_successfully + postgres-metadata: + condition: service_healthy + postgres-training: + condition: service_healthy + + # Airflow init to create admin user + airflow-init: + build: . + user: "0:0" + # image: airflow-optimized:2.9.3 + environment: + <<: *airflow-default-env + volumes: + - ./dags:/opt/airflow/dags + - ./data:/opt/airflow/data + networks: + - cs_dwh + command: > + bash -ceuo pipefail " + mkdir -p /opt/airflow/data && + chmod -R 777 /opt/airflow/data || true && + chown -R airflow:0 /opt/airflow/data || true && + umask 000 && + su -s /bin/bash airflow -c 'airflow db migrate' && + su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' && + su -s /bin/bash airflow -c 'airflow connections delete postgres_training || true' && + su -s /bin/bash airflow -c \"airflow connections add 'postgres_training' --conn-uri \\\"$${AIRFLOW_CONN_POSTGRES_TRAINING}\\\" --conn-description 'Training exercises database'\" + " + depends_on: + postgres-metadata: + condition: service_healthy + prometheus: image: prom/prometheus:v2.53.4 volumes: @@ -127,6 +234,7 @@ networks: volumes: grafana_lib: kafka-data: + pgmeta: superset_data: superset_config: