Первые наброски airflow-gp

2025-09-16 23:03:35 +03:00
parent c23512920b
commit e7e40d18f7
7 changed files with 344 additions and 0 deletions
@@ -0,0 +1,140 @@
+version: "3.9"
+
+services:
+  # Postgres только для Airflow метаданных
+  pgmeta:
+    image: postgres:16
+    # container_name: gp_pgmeta
+    env_file: .env
+    environment:
+      POSTGRES_USER: ${PG_USER}
+      POSTGRES_PASSWORD: ${PG_PASSWORD}
+      POSTGRES_DB: ${PG_DB}
+    ports:
+      - "5433:5432"
+    volumes:
+      - pgmeta:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${PG_USER} -d ${PG_DB}"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+
+  greenplum:
+    image: datagrip/greenplum:6.8
+    # container_name: gp_single
+    hostname: gpdbsne
+    # Порты: внешний 5432
+    ports:
+      - "5432:5432"
+    volumes:
+      - ./sql:/sql:ro
+    # Простая проверка доступности: psql откликается
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 || echo 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+
+  kafka:
+    image: apache/kafka:3.8.0
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
+      KAFKA_LISTENERS: PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
+      # 22-символьный base64 идентификатор для KRaft-кластера
+      CLUSTER_ID: aGVsbG93b3JsZGtpdGNoZW4
+    ports:
+      - "9092:9092"
+
+  kafka-ui:
+    image: provectuslabs/kafka-ui:v0.7.2
+    ports:
+      - 8082:8080
+    environment:
+      - DYNAMIC_CONFIG_ENABLED=true
+      - KAFKA_CLUSTERS_0_NAME=KafkaCluster
+      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:29092
+    volumes:
+      - kafka-ui-data:/app/data
+    depends_on:
+      - kafka
+
+  airflow-webserver:
+    image: apache/airflow:2.9.2
+    container_name: gp_airflow_web
+    env_file: .env
+    environment:
+      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
+    command: >
+      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
+      airflow webserver"
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./airflow/dags:/opt/airflow/dags
+      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+    depends_on:
+      pgmeta:
+        condition: service_healthy
+      kafka:
+        condition: service_started
+      greenplum:
+        condition: service_healthy
+
+  airflow-scheduler:
+    image: apache/airflow:2.9.2
+    container_name: gp_airflow_sch
+    env_file: .env
+    environment:
+      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
+    command: >
+      bash -lc "pip install --no-cache-dir -r /opt/airflow/requirements.txt &&
+      airflow scheduler"
+    volumes:
+      - ./airflow/dags:/opt/airflow/dags
+      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+    depends_on:
+      pgmeta:
+        condition: service_healthy
+      kafka:
+        condition: service_started
+      greenplum:
+        condition: service_healthy
+
+  airflow-init:
+    image: apache/airflow:2.9.2
+    container_name: gp_airflow_init
+    env_file: .env
+    environment:
+      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB}
+    volumes:
+      - ./airflow/dags:/opt/airflow/dags
+      - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+    entrypoint: ["/bin/bash", "-lc"]
+    command: |
+      set -e
+      until pg_isready -h pgmeta -U ${PG_USER} -d ${PG_DB}; do echo 'waiting for pgmeta'; sleep 2; done
+      pip install --no-cache-dir -r /opt/airflow/requirements.txt
+      airflow db migrate
+      airflow users create --username ${AIRFLOW_USER} --password ${AIRFLOW_PASSWORD} --firstname Admin --lastname User --role Admin --email admin@example.org
+    depends_on:
+      pgmeta:
+        condition: service_healthy
+
+volumes:
+  pgmeta:
+  kafka-ui-data: