114 lines
3.6 KiB
YAML
114 lines
3.6 KiB
YAML
services:
|
|
# Postgres только для Airflow метаданных
|
|
pgmeta:
|
|
image: postgres:16
|
|
# container_name: gp_pgmeta
|
|
env_file: .env
|
|
environment:
|
|
POSTGRES_USER: ${PG_USER}
|
|
POSTGRES_PASSWORD: ${PG_PASSWORD}
|
|
POSTGRES_DB: ${PG_DB}
|
|
ports:
|
|
- "5433:5432"
|
|
volumes:
|
|
- pgmeta:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 20
|
|
|
|
greenplum:
|
|
image: woblerr/greenplum:6.27.1
|
|
# container_name: gp_single
|
|
# hostname: gpdbsne
|
|
environment:
|
|
GREENPLUM_USER: ${GP_USER:-gpadmin}
|
|
GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin}
|
|
GREENPLUM_DATABASE_NAME: ${GP_DB:-gpadmin}
|
|
# GP_PORT: ${GP_PORT:-5432}
|
|
# Порты: внешний 5432
|
|
ports:
|
|
- "${GP_PORT}:5432"
|
|
volumes:
|
|
- ./sql:/sql:ro
|
|
- greenplum_data:/data
|
|
# Простая проверка доступности: psql откликается
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gpadmin} || echo 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 30
|
|
|
|
airflow-webserver:
|
|
image: apache/airflow:2.9.2
|
|
container_name: gp_airflow_web
|
|
env_file: .env
|
|
environment:
|
|
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
|
|
command: >
|
|
bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
|
|
airflow webserver"
|
|
ports:
|
|
- "8080:8080"
|
|
volumes:
|
|
- ./airflow/dags:/opt/airflow/dags
|
|
- ./airflow/requirements.txt:/opt/airflow/requirements.txt
|
|
- ./data:/opt/airflow/data
|
|
depends_on:
|
|
pgmeta:
|
|
condition: service_healthy
|
|
greenplum:
|
|
condition: service_healthy
|
|
|
|
airflow-scheduler:
|
|
image: apache/airflow:2.9.2
|
|
container_name: gp_airflow_sch
|
|
env_file: .env
|
|
environment:
|
|
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
|
|
command: >
|
|
bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
|
|
airflow scheduler"
|
|
volumes:
|
|
- ./airflow/dags:/opt/airflow/dags
|
|
- ./airflow/requirements.txt:/opt/airflow/requirements.txt
|
|
- ./data:/opt/airflow/data
|
|
depends_on:
|
|
pgmeta:
|
|
condition: service_healthy
|
|
greenplum:
|
|
condition: service_healthy
|
|
|
|
airflow-init:
|
|
image: apache/airflow:2.9.2
|
|
# container_name: gp_airflow_init
|
|
env_file: .env
|
|
environment:
|
|
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
|
|
volumes:
|
|
- ./airflow/dags:/opt/airflow/dags
|
|
- ./airflow/requirements.txt:/opt/airflow/requirements.txt
|
|
- ./data:/opt/airflow/data
|
|
command: >
|
|
bash -lc "
|
|
set -e;
|
|
pip install --no-cache-dir -r /opt/airflow/requirements.txt;
|
|
# Дожидаемся готовности БД ретрая миграции
|
|
for i in {1..30}; do
|
|
airflow db migrate && break || echo 'waiting for pgmeta' && sleep 3;
|
|
done;
|
|
# Создаём админа; при повторном запуске не падаем
|
|
airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org || true
|
|
"
|
|
depends_on:
|
|
pgmeta:
|
|
condition: service_healthy
|
|
|
|
volumes:
|
|
pgmeta:
|
|
greenplum_data:
|