services: # Postgres только для Airflow метаданных pgmeta: image: postgres:16 # container_name: gp_pgmeta env_file: .env environment: TZ: ${TZ:-Europe/Moscow} POSTGRES_USER: ${PG_USER} POSTGRES_PASSWORD: ${PG_PASSWORD} POSTGRES_DB: ${PG_DB} ports: - "5433:5432" volumes: - pgmeta:/var/lib/postgresql/data healthcheck: test: [ "CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}" ] interval: 5s timeout: 5s retries: 20 # Postgres с демо-БД bookings (источник для будущего DWH) bookings-db: image: postgres:16 env_file: .env environment: TZ: ${TZ:-Europe/Moscow} POSTGRES_USER: ${BOOKINGS_DB_USER} POSTGRES_PASSWORD: ${BOOKINGS_DB_PASSWORD} POSTGRES_DB: ${BOOKINGS_DB_NAME} ports: - "${BOOKINGS_DB_PORT:-5434}:5432" volumes: - bookings_data:/var/lib/postgresql/data # В этот каталог будет монтироваться генератор demodb - ./bookings:/bookings:ro healthcheck: test: [ "CMD-SHELL", "pg_isready -U ${BOOKINGS_DB_USER} -d ${BOOKINGS_DB_NAME}" ] interval: 5s timeout: 5s retries: 20 greenplum: image: woblerr/greenplum:6.27.1 # based on https://github.com/woblerr/docker-greenplum # container_name: gp_single # hostname: gpdbsne environment: TZ: ${TZ:-Europe/Moscow} GREENPLUM_USER: ${GP_USER:-gpadmin} GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin} GREENPLUM_DATABASE_NAME: ${GP_DB:-gp_dwh} GREENPLUM_PXF_ENABLE: "true" # Порты: внешний 5435 (на хосте) ports: - "5435:5432" volumes: - ./sql:/sql:ro - ./pxf/postgresql-42.7.3.jar:/pxf-local/postgresql-42.7.3.jar:ro - ./pxf/servers/bookings-db:/pxf-local/servers/bookings-db:ro - ./pxf/init/10_pxf_bookings.sh:/docker-entrypoint-initdb.d/10_pxf_bookings.sh:ro - greenplum_data:/data # Простая проверка доступности: psql откликается healthcheck: test: [ "CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gp_dwh} || echo 1" ] interval: 10s timeout: 5s retries: 30 airflow-webserver: build: . image: airflow-custom:latest container_name: gp_airflow_web env_file: .env environment: TZ: ${TZ:-Europe/Moscow} AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} AIRFLOW_CONN_GREENPLUM_CONN: postgresql://${GP_USER}:${GP_PASSWORD}@greenplum:${GP_PORT:-5432}/${GP_DB} AIRFLOW_CONN_BOOKINGS_DB: postgresql://${BOOKINGS_DB_USER}:${BOOKINGS_DB_PASSWORD}@bookings-db:5432/demo command: airflow webserver ports: - "8080:8080" volumes: - ./airflow/dags:/opt/airflow/dags - ./airflow/requirements.txt:/opt/airflow/requirements.txt - ./sql:/sql:ro - airflow_data:/opt/airflow/data depends_on: pgmeta: condition: service_healthy greenplum: condition: service_healthy airflow-init: condition: service_completed_successfully airflow-scheduler: build: . image: airflow-custom:latest container_name: gp_airflow_sch env_file: .env environment: TZ: ${TZ:-Europe/Moscow} AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} AIRFLOW_CONN_GREENPLUM_CONN: postgresql://${GP_USER}:${GP_PASSWORD}@greenplum:${GP_PORT:-5432}/${GP_DB} AIRFLOW_CONN_BOOKINGS_DB: postgresql://${BOOKINGS_DB_USER}:${BOOKINGS_DB_PASSWORD}@bookings-db:5432/demo command: airflow scheduler volumes: - ./airflow/dags:/opt/airflow/dags - ./airflow/requirements.txt:/opt/airflow/requirements.txt - ./sql:/sql:ro - airflow_data:/opt/airflow/data depends_on: pgmeta: condition: service_healthy greenplum: condition: service_healthy airflow-init: condition: service_completed_successfully airflow-init: build: . image: airflow-custom:latest # container_name: gp_airflow_init user: "0" env_file: .env environment: TZ: ${TZ:-Europe/Moscow} AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} AIRFLOW_CONN_GREENPLUM_CONN: postgresql://${GP_USER}:${GP_PASSWORD}@greenplum:${GP_PORT:-5432}/${GP_DB} AIRFLOW_CONN_BOOKINGS_DB: postgresql://${BOOKINGS_DB_USER}:${BOOKINGS_DB_PASSWORD}@bookings-db:5432/demo volumes: - ./airflow/dags:/opt/airflow/dags - ./airflow/requirements.txt:/opt/airflow/requirements.txt - ./sql:/sql:ro - airflow_data:/opt/airflow/data command: > bash -lc " set -e; mkdir -p /opt/airflow/data && chown -R airflow:root /opt/airflow/data; # Дожидаемся готовности БД ретрая миграции for i in {1..30}; do su -s /bin/bash airflow -c \"PATH='/home/airflow/.local/bin:$${PATH}' airflow db migrate\" && break || echo 'waiting for pgmeta' && sleep 3; done; # Создаём админа; при повторном запуске не падаем su -s /bin/bash airflow -c \"PATH='/home/airflow/.local/bin:$${PATH}' airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org\" || true " depends_on: pgmeta: condition: service_healthy volumes: pgmeta: bookings_data: greenplum_data: airflow_data: