Убрана kafka, переделан dag на работу с csv

2025-10-14 22:43:20 +03:00
parent 50865db5e6
commit d17c1da542
8 changed files with 182 additions and 220 deletions
@@ -16,10 +16,6 @@ GP_PORT=5432
 GP_CONN_ID=greenplum_conn
 GP_USE_AIRFLOW_CONN=true

-# Kafka Configuration
-KAFKA_BOOTSTRAP=kafka:29092
-KAFKA_TOPIC=orders
-KAFKA_BATCH_SIZE=500
-KAFKA_POLL_TIMEOUT=10
-KAFKA_MAX_EMPTY_POLLS=3
-KAFKA_UI_CLUSTER_NAME=KafkaCluster
+# CSV Pipeline
+CSV_DIR=/opt/airflow/data
+CSV_ROWS=1000