de-roadmap/airflow-greenplum/docker-compose.yml

services:
  # Postgres только для Airflow метаданных
  pgmeta:
    image: postgres:16
    # container_name: gp_pgmeta
    env_file: .env
    environment:
      POSTGRES_USER: ${PG_USER}
      POSTGRES_PASSWORD: ${PG_PASSWORD}
      POSTGRES_DB: ${PG_DB}
    ports:
      - "5433:5432"
    volumes:
      - pgmeta:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}"]
      interval: 5s
      timeout: 5s
      retries: 20

  greenplum:
    image: woblerr/greenplum:6.27.1
    # container_name: gp_single
    # hostname: gpdbsne
    environment:
      GREENPLUM_USER: ${GP_USER:-gpadmin}
      GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin}
      GREENPLUM_DATABASE_NAME: ${GP_DB:-gpadmin}
      # GP_PORT: ${GP_PORT:-5432}
    # Порты: внешний 5432
    ports:
      - "${GP_PORT}:5432"
    volumes:
      - ./sql:/sql:ro
      - greenplum_data:/data
    # Простая проверка доступности: psql откликается
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gpadmin} || echo 1"]
      interval: 10s
      timeout: 5s
      retries: 30

  kafka:
    image: apache/kafka:3.8.0
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
      KAFKA_LISTENERS: PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
      # 22-символьный base64 идентификатор для KRaft-кластера
      CLUSTER_ID: aGVsbG93b3JsZGtpdGNoZW4
    ports:
      - "9092:9092"

  kafka-ui:
    image: provectuslabs/kafka-ui:v0.7.2
    ports:
      - 8082:8080
    environment:
      - DYNAMIC_CONFIG_ENABLED=true
      - KAFKA_CLUSTERS_0_NAME=KafkaCluster
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:29092
    volumes:
      - kafka-ui-data:/app/data
    depends_on:
      - kafka

  airflow-webserver:
    image: apache/airflow:2.9.2
    container_name: gp_airflow_web
    env_file: .env
    environment:
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
    command: >
      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
      airflow webserver"
    ports:
      - "8080:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
    depends_on:
      pgmeta:
        condition: service_healthy
      kafka:
        condition: service_started
      greenplum:
        condition: service_healthy

  airflow-scheduler:
    image: apache/airflow:2.9.2
    container_name: gp_airflow_sch
    env_file: .env
    environment:
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
    command: >
      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
      airflow scheduler"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
    depends_on:
      pgmeta:
        condition: service_healthy
      kafka:
        condition: service_started
      greenplum:
        condition: service_healthy

  airflow-init:
    image: apache/airflow:2.9.2
    # container_name: gp_airflow_init
    env_file: .env
    environment:
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
    command: >
      bash -lc "
        set -e;
        pip install --no-cache-dir -r /opt/airflow/requirements.txt;
        # Дожидаемся готовности БД ретрая миграции
        for i in {1..30}; do
          airflow db migrate && break || echo 'waiting for pgmeta' && sleep 3;
        done;
        # Создаём админа; при повторном запуске не падаем
        airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org || true
      "
    depends_on:
      pgmeta:
        condition: service_healthy

volumes:
  pgmeta:
  kafka-ui-data:
  greenplum_data: