Убрана kafka, переделан dag на работу с csv

airflow-greenplum/docker-compose.yml

@@ -40,42 +40,6 @@ services:
       timeout: 5s
       retries: 30
 
-  kafka:
-    image: apache/kafka:3.8.0
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_PROCESS_ROLES: broker,controller
-      KAFKA_NODE_ID: 1
-      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
-      KAFKA_LISTENERS: PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
-      # 22-символьный base64 идентификатор для KRaft-кластера
-      CLUSTER_ID: aGVsbG93b3JsZGtpdGNoZW4
-    ports:
-      - "9092:9092"
-
-  kafka-ui:
-    image: provectuslabs/kafka-ui:v0.7.2
-    env_file: .env
-    ports:
-      - 8082:8080
-    environment:
-      DYNAMIC_CONFIG_ENABLED: "true"
-      KAFKA_CLUSTERS_0_NAME: ${KAFKA_UI_CLUSTER_NAME:-KafkaCluster}
-      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: ${KAFKA_BOOTSTRAP:-kafka:29092}
-    volumes:
-      - kafka-ui-data:/app/data
-    depends_on:
-      - kafka
-
   airflow-webserver:
     image: apache/airflow:2.9.2
     container_name: gp_airflow_web
@@ -91,11 +55,10 @@ services:
     volumes:
       - ./airflow/dags:/opt/airflow/dags
       - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+      - ./data:/opt/airflow/data
     depends_on:
       pgmeta:
         condition: service_healthy
-      kafka:
-        condition: service_started
       greenplum:
         condition: service_healthy
 
@@ -112,11 +75,10 @@ services:
     volumes:
       - ./airflow/dags:/opt/airflow/dags
       - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+      - ./data:/opt/airflow/data
     depends_on:
       pgmeta:
         condition: service_healthy
-      kafka:
-        condition: service_started
       greenplum:
         condition: service_healthy
 
@@ -130,6 +92,7 @@ services:
     volumes:
       - ./airflow/dags:/opt/airflow/dags
       - ./airflow/requirements.txt:/opt/airflow/requirements.txt
+      - ./data:/opt/airflow/data
     command: >
       bash -lc "
         set -e;
@@ -147,5 +110,4 @@ services:
 
 volumes:
   pgmeta:
-  kafka-ui-data:
   greenplum_data: