x-airflow-common-env: &airflow-env TZ: ${TZ:-Europe/Moscow} AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} AIRFLOW_CONN_GREENPLUM_CONN: postgresql://${GP_USER}:${GP_PASSWORD}@greenplum:${GP_PORT:-5432}/${GP_DB} AIRFLOW_CONN_BOOKINGS_DB: postgresql://${BOOKINGS_DB_USER}:${BOOKINGS_DB_PASSWORD}@bookings-db:5432/demo x-airflow-common-volumes: &airflow-volumes - ./airflow/dags:/opt/airflow/dags - ./sql:/sql:ro - airflow_data:/opt/airflow/data x-airflow-common-depends: &airflow-depends pgmeta: condition: service_healthy greenplum: condition: service_healthy airflow-init: condition: service_completed_successfully services: # Postgres только для Airflow метаданных pgmeta: image: postgres:16 # container_name: gp_pgmeta env_file: .env environment: TZ: ${TZ:-Europe/Moscow} POSTGRES_USER: ${PG_USER} POSTGRES_PASSWORD: ${PG_PASSWORD} POSTGRES_DB: ${PG_DB} ports: - "5433:5432" volumes: - pgmeta:/var/lib/postgresql/data healthcheck: test: [ "CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}" ] interval: 5s timeout: 5s retries: 20 # Postgres с демо-БД bookings (источник для будущего DWH) bookings-db: image: postgres:16 env_file: .env environment: TZ: ${TZ:-Europe/Moscow} POSTGRES_USER: ${BOOKINGS_DB_USER} POSTGRES_PASSWORD: ${BOOKINGS_DB_PASSWORD} POSTGRES_DB: ${BOOKINGS_DB_NAME} ports: - "${BOOKINGS_DB_PORT:-5434}:5432" volumes: - bookings_data:/var/lib/postgresql/data # В этот каталог будет монтироваться генератор demodb - ./bookings:/bookings:ro healthcheck: test: [ "CMD-SHELL", "pg_isready -U ${BOOKINGS_DB_USER} -d ${BOOKINGS_DB_NAME}" ] interval: 5s timeout: 5s retries: 20 greenplum: build: context: . dockerfile: Dockerfile.greenplum image: greenplum-custom:6.27.1 # container_name: gp_single hostname: gpdbsne environment: TZ: ${TZ:-Europe/Moscow} GREENPLUM_USER: ${GP_USER:-gpadmin} GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin} GREENPLUM_DATABASE_NAME: ${GP_DB:-gp_dwh} GREENPLUM_PXF_ENABLE: "true" PXF_SEED_OVERWRITE: ${PXF_SEED_OVERWRITE:-0} PXF_SYNC_ON_START: ${PXF_SYNC_ON_START:-0} # Порты: внешний 5435 (на хосте) ports: - "5435:5432" volumes: - ./sql:/sql:ro - greenplum_data:/data # Простая проверка доступности: psql откликается healthcheck: # Ждём не только доступность GPDB, но и готовность PXF, # чтобы Airflow не стартовал раньше PXF. test: [ "CMD-SHELL", "/usr/local/greenplum-db/bin/pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gp_dwh} && PGPASSWORD=${GP_PASSWORD:-gpadmin} /usr/local/greenplum-db/bin/psql -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gp_dwh} -t -A -c \"SELECT 1 FROM pg_extension WHERE extname='pxf';\" | grep -q 1 && su - ${GP_USER:-gpadmin} -c '/usr/local/pxf/bin/pxf cluster status' >/dev/null 2>&1" ] interval: 10s timeout: 5s retries: 30 start_period: 40s airflow-webserver: build: context: . dockerfile: Dockerfile.airflow image: airflow-custom:latest container_name: gp_airflow_webserver env_file: .env environment: <<: *airflow-env command: airflow webserver ports: - "8080:8080" volumes: - ./airflow/dags:/opt/airflow/dags - ./sql:/sql:ro - airflow_data:/opt/airflow/data - ./airflow/requirements.txt:/opt/airflow/requirements.txt healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/health"] interval: 30s timeout: 10s retries: 5 start_period: 40s depends_on: <<: *airflow-depends airflow-scheduler: build: context: . dockerfile: Dockerfile.airflow image: airflow-custom:latest container_name: gp_airflow_scheduler env_file: .env environment: <<: *airflow-env command: airflow scheduler volumes: - ./airflow/dags:/opt/airflow/dags - ./sql:/sql:ro - airflow_data:/opt/airflow/data - ./airflow/requirements.txt:/opt/airflow/requirements.txt depends_on: <<: *airflow-depends airflow-init: build: context: . dockerfile: Dockerfile.airflow image: airflow-custom:latest # Имя контейнера закомментировано (одноразовый сервис) user: "0" env_file: .env environment: <<: *airflow-env volumes: - ./airflow/dags:/opt/airflow/dags - ./sql:/sql:ro - airflow_data:/opt/airflow/data command: > bash -lc " set -e; mkdir -p /opt/airflow/data && chown -R airflow:root /opt/airflow/data; # Дожидаемся готовности БД ретрая миграции for i in {1..30}; do su -s /bin/bash airflow -c \"PATH='/home/airflow/.local/bin:$${PATH}' airflow db migrate\" && break || echo 'waiting for pgmeta' && sleep 3; done; # Создаём админа; при повторном запуске не падаем su -s /bin/bash airflow -c \"PATH='/home/airflow/.local/bin:$${PATH}' airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org\" || true " depends_on: pgmeta: condition: service_healthy volumes: pgmeta: bookings_data: greenplum_data: airflow_data: