airflow-greenplum/docker-compose.yml

x-airflow-common-env: &airflow-env
  TZ: ${TZ:-Europe/Moscow}
  AIRFLOW__CORE__LOAD_EXAMPLES: "False"
  AIRFLOW__CORE__EXECUTOR: LocalExecutor
  AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
  AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
  AIRFLOW_CONN_GREENPLUM_CONN: postgresql://${GP_USER}:${GP_PASSWORD}@greenplum:${GP_PORT:-5432}/${GP_DB}
  AIRFLOW_CONN_BOOKINGS_DB: postgresql://${BOOKINGS_DB_USER}:${BOOKINGS_DB_PASSWORD}@bookings-db:5432/demo

x-airflow-common-volumes: &airflow-volumes
  - ./airflow/dags:/opt/airflow/dags
  - ./sql:/sql:ro
  - airflow_data:/opt/airflow/data
  - airflow_logs:/opt/airflow/logs

x-airflow-common-depends: &airflow-depends
  pgmeta:
    condition: service_healthy
  greenplum:
    condition: service_healthy
  airflow-init:
    condition: service_completed_successfully

services:
  # Postgres только для Airflow метаданных
  pgmeta:
    image: postgres:16
    # container_name: gp_pgmeta
    env_file: .env
    environment:
      TZ: ${TZ:-Europe/Moscow}
      POSTGRES_USER: ${PG_USER}
      POSTGRES_PASSWORD: ${PG_PASSWORD}
      POSTGRES_DB: ${PG_DB}
    ports:
      - "${PGMETA_PORT:-5433}:5432"
    volumes:
      - pgmeta:/var/lib/postgresql/data
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}" ]
      interval: 5s
      timeout: 5s
      retries: 20

  # Postgres с демо-БД bookings (источник для будущего DWH)
  bookings-db:
    image: postgres:16
    env_file: .env
    environment:
      TZ: ${TZ:-Europe/Moscow}
      POSTGRES_USER: ${BOOKINGS_DB_USER}
      POSTGRES_PASSWORD: ${BOOKINGS_DB_PASSWORD}
      POSTGRES_DB: ${BOOKINGS_DB_NAME}
    ports:
      - "${BOOKINGS_DB_PORT:-5434}:5432"
    volumes:
      - bookings_data:/var/lib/postgresql/data
      # В этот каталог будет монтироваться генератор demodb
      - ./bookings:/bookings:ro
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready -U ${BOOKINGS_DB_USER} -d ${BOOKINGS_DB_NAME}" ]
      interval: 5s
      timeout: 5s
      retries: 20

  greenplum:
    build:
      context: .
      dockerfile: Dockerfile.greenplum
    image: greenplum-custom:6.27.1
    # container_name: gp_single
    hostname: gpdbsne
    environment:
      TZ: ${TZ:-Europe/Moscow}
      GREENPLUM_USER: ${GP_USER:-gpadmin}
      GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin}
      GREENPLUM_DATABASE_NAME: ${GP_DB:-gp_dwh}
      GREENPLUM_PXF_ENABLE: "true"
      PXF_SEED_OVERWRITE: ${PXF_SEED_OVERWRITE:-0}
      PXF_SYNC_ON_START: ${PXF_SYNC_ON_START:-0}
    # Порты: внешний 5435 (на хосте)
    ports:
      - "${GP_HOST_PORT:-5435}:5432"
    volumes:
      - ./sql:/sql:ro
      - greenplum_data:/data
    # Простая проверка доступности: psql откликается
    healthcheck:
      # Ждём не только доступность GPDB, но и готовность PXF,
      # чтобы Airflow не стартовал раньше PXF.
      test: [ "CMD-SHELL", "/usr/local/greenplum-db/bin/pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gp_dwh} && PGPASSWORD=${GP_PASSWORD:-gpadmin} /usr/local/greenplum-db/bin/psql -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gp_dwh} -t -A -c \"SELECT 1 FROM pg_extension WHERE extname='pxf';\" | grep -q 1 && su - ${GP_USER:-gpadmin} -c '/usr/local/pxf/bin/pxf cluster status' >/dev/null 2>&1" ]
      interval: 10s
      timeout: 5s
      retries: 30
      start_period: 40s

  airflow-webserver:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    image: airflow-custom:latest
    container_name: gp_airflow_webserver
    env_file: .env
    environment:
      <<: *airflow-env
    command: bash -lc "rm -f /opt/airflow/airflow-webserver.pid && exec airflow webserver"
    ports:
      - "${AIRFLOW_WEB_PORT:-8080}:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./sql:/sql:ro
      - airflow_data:/opt/airflow/data
      - airflow_logs:/opt/airflow/logs
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 40s
    depends_on:
      <<: *airflow-depends

  airflow-scheduler:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    image: airflow-custom:latest
    container_name: gp_airflow_scheduler
    env_file: .env
    environment:
      <<: *airflow-env
    command: airflow scheduler
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./sql:/sql:ro
      - airflow_data:/opt/airflow/data
      - airflow_logs:/opt/airflow/logs
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
    depends_on:
      <<: *airflow-depends

  airflow-init:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    image: airflow-custom:latest
    # Имя контейнера закомментировано (одноразовый сервис)
    user: "0"
    env_file: .env
    environment:
      <<: *airflow-env
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./sql:/sql:ro
      - airflow_data:/opt/airflow/data
      - airflow_logs:/opt/airflow/logs
    command: >
      bash -lc "
        set -e;
        mkdir -p /opt/airflow/data /opt/airflow/logs && chown -R airflow:root /opt/airflow/data /opt/airflow/logs;
        # Дожидаемся готовности БД ретрая миграции
        for i in {1..30}; do
          su -s /bin/bash airflow -c \"PATH='/home/airflow/.local/bin:$${PATH}' airflow db migrate\" && break || echo 'waiting for pgmeta' && sleep 3;
        done;
        # Создаём админа; при повторном запуске не падаем
        su -s /bin/bash airflow -c \"PATH='/home/airflow/.local/bin:$${PATH}' airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org\" || true
      "
    depends_on:
      pgmeta:
        condition: service_healthy

volumes:
  pgmeta:
  bookings_data:
  greenplum_data:
  airflow_data:
  airflow_logs: