ddadmin
fe9c15c0fe
Update Airflow configuration to integrate with ClickHouse DWH instead of PostgreSQL training database. Changes include: - Switch Airflow dependencies from PostgreSQL to ClickHouse connector - Update docker-compose to use ClickHouse connection and correct Dockerfile - Refactor airflow/requirements.txt to include only essential packages - Add DAGs directory for ETL pipeline orchestration - Update documentation to reflect Airflow integration and access credentials - Adjust service dependencies to wait for ClickHouse startup
47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
"""
|
|
ETL Pipeline DAG для ClickHouse DWH
|
|
|
|
Шаблон DAG для оркестрации пайплайна данных.
|
|
Полная реализация будет добавлена позже.
|
|
|
|
Пайплайн:
|
|
1. DDL - создание структуры БД
|
|
2. Load - загрузка данных в Kafka
|
|
3. Transform - batch трансформация ODS → DDS → DM
|
|
"""
|
|
|
|
from datetime import datetime, timedelta
|
|
from airflow import DAG
|
|
from airflow.operators.bash import BashOperator
|
|
from airflow.operators.empty import EmptyOperator
|
|
|
|
# Базовые настройки DAG
|
|
default_args = {
|
|
"owner": "airflow",
|
|
"depends_on_past": False,
|
|
"email_on_failure": False,
|
|
"email_on_retry": False,
|
|
"retries": 1,
|
|
"retry_delay": timedelta(minutes=5),
|
|
}
|
|
|
|
with DAG(
|
|
dag_id="etl_pipeline",
|
|
default_args=default_args,
|
|
description="ETL pipeline для ClickHouse DWH",
|
|
schedule=None, # Запуск только вручную (пока)
|
|
start_date=datetime(2024, 1, 1),
|
|
catchup=False,
|
|
tags=["etl", "clickhouse", "dwh"],
|
|
) as dag:
|
|
|
|
# TODO: добавить задачи пайплайна
|
|
# - ddl: создание структуры БД
|
|
# - load: загрузка данных в Kafka
|
|
# - transform: batch трансформация
|
|
|
|
start = EmptyOperator(task_id="start")
|
|
end = EmptyOperator(task_id="end")
|
|
|
|
start >> end
|