Первая публикация кода

2025-10-17 19:16:15 +03:00
commit 0dffd5df20
17 changed files with 3440 additions and 0 deletions
@@ -0,0 +1,113 @@
+services:
+  # Postgres только для Airflow метаданных
+  pgmeta:
+    image: postgres:16
+    # container_name: gp_pgmeta
+    env_file: .env
+    environment:
+      POSTGRES_USER: ${PG_USER}
+      POSTGRES_PASSWORD: ${PG_PASSWORD}
+      POSTGRES_DB: ${PG_DB}
+    ports:
+      - "5433:5432"
+    volumes:
+      - pgmeta:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+
+  greenplum:
+    image: woblerr/greenplum:6.27.1
+    # container_name: gp_single
+    # hostname: gpdbsne
+    environment:
+      GREENPLUM_USER: ${GP_USER:-gpadmin}
+      GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin}
+      GREENPLUM_DATABASE_NAME: ${GP_DB:-gpadmin}
+      # GP_PORT: ${GP_PORT:-5432}
+    # Порты: внешний 5432
+    ports:
+      - "${GP_PORT}:5432"
+    volumes:
+      - ./sql:/sql:ro
+      - greenplum_data:/data
+    # Простая проверка доступности: psql откликается
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gpadmin} || echo 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+
+  airflow-webserver:
+    image: apache/airflow:2.9.2
+    container_name: gp_airflow_web
+    env_file: .env
+    environment:
+      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
+    command: >
+      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
+      airflow webserver"
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./airflow/dags:/opt/airflow/dags
+      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+      - ./data:/opt/airflow/data
+    depends_on:
+      pgmeta:
+        condition: service_healthy
+      greenplum:
+        condition: service_healthy
+
+  airflow-scheduler:
+    image: apache/airflow:2.9.2
+    container_name: gp_airflow_sch
+    env_file: .env
+    environment:
+      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
+    command: >
+      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
+      airflow scheduler"
+    volumes:
+      - ./airflow/dags:/opt/airflow/dags
+      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+      - ./data:/opt/airflow/data
+    depends_on:
+      pgmeta:
+        condition: service_healthy
+      greenplum:
+        condition: service_healthy
+
+  airflow-init:
+    image: apache/airflow:2.9.2
+    # container_name: gp_airflow_init
+    env_file: .env
+    environment:
+      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
+    volumes:
+      - ./airflow/dags:/opt/airflow/dags
+      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+      - ./data:/opt/airflow/data
+    command: >
+      bash -lc "
+        set -e;
+        pip install --no-cache-dir -r /opt/airflow/requirements.txt;
+        # Дожидаемся готовности БД ретрая миграции
+        for i in {1..30}; do
+          airflow db migrate && break || echo 'waiting for pgmeta' && sleep 3;
+        done;
+        # Создаём админа; при повторном запуске не падаем
+        airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org || true
+      "
+    depends_on:
+      pgmeta:
+        condition: service_healthy
+
+volumes:
+  pgmeta:
+  greenplum_data: