airflow-greenplum/docker-compose.yml

services:
  # Postgres только для Airflow метаданных
  pgmeta:
    image: postgres:16
    # container_name: gp_pgmeta
    env_file: .env
    environment:
      TZ: ${TZ:-Europe/Moscow}
      POSTGRES_USER: ${PG_USER}
      POSTGRES_PASSWORD: ${PG_PASSWORD}
      POSTGRES_DB: ${PG_DB}
    ports:
      - "5433:5432"
    volumes:
      - pgmeta:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}"]
      interval: 5s
      timeout: 5s
      retries: 20

  # Postgres с демо-БД bookings (источник для будущего DWH)
  bookings-db:
    image: postgres:16
    env_file: .env
    environment:
      TZ: ${TZ:-Europe/Moscow}
      POSTGRES_USER: ${BOOKINGS_DB_USER}
      POSTGRES_PASSWORD: ${BOOKINGS_DB_PASSWORD}
      POSTGRES_DB: ${BOOKINGS_DB_NAME}
    ports:
      - "${BOOKINGS_DB_PORT:-5434}:5432"
    volumes:
      - bookings_data:/var/lib/postgresql/data
      # В этот каталог будет монтироваться генератор demodb
      - ./bookings:/bookings:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${BOOKINGS_DB_USER} -d ${BOOKINGS_DB_NAME}"]
      interval: 5s
      timeout: 5s
      retries: 20

  greenplum:
    image: woblerr/greenplum:6.27.1  # based on https://github.com/woblerr/docker-greenplum
    # container_name: gp_single
    # hostname: gpdbsne
    environment:
      TZ: ${TZ:-Europe/Moscow}
      GREENPLUM_USER: ${GP_USER:-gpadmin}
      GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin}
      GREENPLUM_DATABASE_NAME: ${GP_DB:-gp_dwh}
      GREENPLUM_PXF_ENABLE: "true"
    # Порты: внешний 5435 (на хосте)
    ports:
      - "5435:5432"
    volumes:
      - ./sql:/sql:ro
      - ./pxf/postgresql-42.7.3.jar:/pxf-local/postgresql-42.7.3.jar:ro
      - ./pxf/servers/bookings-db:/pxf-local/servers/bookings-db:ro
      - ./pxf/init/10_pxf_bookings.sh:/docker-entrypoint-initdb.d/10_pxf_bookings.sh:ro
      - greenplum_data:/data
      - airflow_data:/opt/airflow/data
    # Простая проверка доступности: psql откликается
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gp_dwh} || echo 1"]
      interval: 10s
      timeout: 5s
      retries: 30

  airflow-webserver:
    image: apache/airflow:2.9.2
    container_name: gp_airflow_web
    env_file: .env
    environment:
      TZ: ${TZ:-Europe/Moscow}
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
      AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
    command: >
      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
      airflow webserver"
    ports:
      - "8080:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
      - ./sql:/sql:ro
      - airflow_data:/opt/airflow/data
    depends_on:
      pgmeta:
        condition: service_healthy
      greenplum:
        condition: service_healthy

  airflow-scheduler:
    image: apache/airflow:2.9.2
    container_name: gp_airflow_sch
    env_file: .env
    environment:
      TZ: ${TZ:-Europe/Moscow}
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
      AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
    command: >
      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
      airflow scheduler"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
      - ./sql:/sql:ro
      - airflow_data:/opt/airflow/data
    depends_on:
      pgmeta:
        condition: service_healthy
      greenplum:
        condition: service_healthy

  airflow-init:
    image: apache/airflow:2.9.2
    # container_name: gp_airflow_init
    env_file: .env
    environment:
      TZ: ${TZ:-Europe/Moscow}
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
      AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
    user: "0:0"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
      - ./sql:/sql:ro
      - airflow_data:/opt/airflow/data
    command: >
      bash -lc "
        set -e;
        # mkdir -p /opt/airflow/data && chmod -R 777 /opt/airflow/data;
        # su - airflow;
        pip install --no-cache-dir -r /opt/airflow/requirements.txt;
        # Дожидаемся готовности БД ретрая миграции
        for i in {1..30}; do
          airflow db migrate && break || echo 'waiting for pgmeta' && sleep 3;
        done;
        # Создаём админа; при повторном запуске не падаем
        airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org || true
      "
    depends_on:
      pgmeta:
        condition: service_healthy

volumes:
  pgmeta:
  bookings_data:
  greenplum_data:
  airflow_data: