clickstream-ch-kafka-superset-demo/docker-compose.yml

x-airflow-env: &airflow-default-env
  AIRFLOW__CORE__LOAD_EXAMPLES: "False"
  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
  AIRFLOW__CORE__EXECUTOR: LocalExecutor
  AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW_SECRET_KEY:-replace-me-with-random-string}
  # ClickHouse connection для ETL
  AIRFLOW_CONN_CLICKHOUSE_DEFAULT: clickhouse://default:123456@clickhouse:9000/default
  # StatsD для мониторинга (метрики → statsd-exporter → Prometheus)
  AIRFLOW__METRICS__STATSD_ON: "True"
  AIRFLOW__METRICS__STATSD_HOST: "statsd-exporter"
  AIRFLOW__METRICS__STATSD_PORT: "8125"
  AIRFLOW__METRICS__STATSD_PREFIX: "airflow"


services:
  clickhouse:
    image: clickhouse/clickhouse-server:25.1
    environment:
        TZ: "Europe/Moscow"
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144
    networks:
      - cs_dwh
    volumes:
      - clickhouse-data:/var/lib/clickhouse
      - ./configs/default_user.xml:/etc/clickhouse-server/users.d/default_user.xml
      # - ./configs/z_config.xml:/etc/clickhouse-server/config.d/z_config.xml
      # - ./configs/macros_ch1.xml:/etc/clickhouse-server/config.d/macros.xml
      # - ./configs/clickhouse-backup.yml:/etc/clickhouse-backup/config.yml
      # - ./configs/s3-storage.xml:/etc/clickhouse-server/config.d/s3-storage.xml
      - ./configs/prometheus_ch.xml:/etc/clickhouse-server/config.d/s3-prometheus_ch.xml
    ports:
      - "8002:9000"
      - "9123:8123"

  kafka:
    image: apache/kafka:3.8.0
    # hostname: kafka
    # container_name: kafka
    ports:
      - 9092:9092
    # Данные топиков/сообщений сохраняются в named volume `kafka-data` (между `docker compose down/up`).
    volumes:
      - kafka-data:/var/lib/kafka/data
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
      KAFKA_LISTENERS: PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
    networks:
      - cs_dwh

  kafka-ui:
    # container_name: kafka-ui
    image: provectuslabs/kafka-ui:v0.7.2
    ports:
      - 8082:8080
    depends_on:
      - kafka
    environment:
      DYNAMIC_CONFIG_ENABLED: "true"
      KAFKA_CLUSTERS_0_NAME: "Kafka Cluster"
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka:29092"
      KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: "PLAINTEXT"
    networks:
      - cs_dwh

  # PostgreSQL for Airflow metadata
  postgres-metadata:
    image: postgres:16
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    ports:
      - "5434:5432"
    volumes:
      - pgmeta:/var/lib/postgresql/data
    networks:
      - cs_dwh
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U airflow -d airflow"]
      interval: 5s
      timeout: 5s
      retries: 20

  # Airflow webserver
  airflow-webserver:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    image: airflow-optimized:2.10.5
    environment:
      <<: *airflow-default-env
    command: >
      bash -c "
        airflow webserver
      "
    ports:
      - "8080:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./sql:/opt/airflow/sql:ro
      - ./data:/opt/airflow/data
    networks:
      - cs_dwh
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres-metadata:
        condition: service_healthy
      clickhouse:
        condition: service_started

  # Airflow scheduler
  airflow-scheduler:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    environment:
      <<: *airflow-default-env
    command: >
      bash -c "
        airflow scheduler
      "
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./sql:/opt/airflow/sql:ro
      - ./data:/opt/airflow/data
    networks:
      - cs_dwh
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres-metadata:
        condition: service_healthy
      clickhouse:
        condition: service_started

  # Airflow init to create admin user
  airflow-init:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    user: "0:0"
    environment:
      <<: *airflow-default-env
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./sql:/opt/airflow/sql:ro
      - ./data:/opt/airflow/data
    networks:
      - cs_dwh
    command: >
      bash -ceuo pipefail "
        mkdir -p /opt/airflow/data &&
        chmod -R a+rX /opt/airflow/data || true &&
        chown -R airflow:0 /opt/airflow/data || true &&
        umask 000 &&
        su -s /bin/bash airflow -c 'airflow db migrate' &&
        su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' &&
        su -s /bin/bash airflow -c 'airflow connections delete clickhouse_default || true' &&
        su -s /bin/bash airflow -c \"airflow connections add 'clickhouse_default' --conn-uri \\\"$${AIRFLOW_CONN_CLICKHOUSE_DEFAULT}\\\" --conn-description 'ClickHouse DWH'\"
      "
    depends_on:
      postgres-metadata:
        condition: service_healthy

  prometheus:
    image: prom/prometheus:v2.53.4
    volumes:
      - ./configs/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    # container_name: prometheus
    # hostname: prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
    ports:
      - 9090:9090
    restart: unless-stopped
    environment:
      TZ: "Europe/Moscow"
    networks:
      - cs_dwh

  grafana:
    image: grafana/grafana:11.5.2-ubuntu
    depends_on:
      - prometheus
    ports:
      - 3000:3000
    volumes:
      - grafana_lib:/var/lib/grafana
      - ./configs/grafana/provisioning/:/etc/grafana/provisioning/:ro
    # container_name: grafana
    # hostname: grafana
    restart: unless-stopped
    environment:
      TZ: "Europe/Moscow"
    networks:
      - cs_dwh

  superset:
    # image: apache/superset
    build: 
      context: .
      dockerfile: Dockerfile.superset
    restart: unless-stopped
    ports:
      - "8088:8088"
    networks:
      - cs_dwh
    volumes:
      - superset_data:/var/lib/superset
      - superset_config:/app/superset_home
    environment:
      - SUPERSET_SECRET_KEY=9wc5+erMt60+lxrXDf3RjeIR+zONpEFusO00Np7JzfliMTI1e+RXnHcQ
      - TZ=Europe/Moscow
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
      interval: 30s
      timeout: 10s
      retries: 5

  # Kafka Exporter для мониторинга через Prometheus
  kafka-exporter:
    image: danielqsj/kafka-exporter:v1.9.0
    command: ["--kafka.server=kafka:29092"]
    ports:
      - "9308:9308"
    networks:
      - cs_dwh
    depends_on:
      - kafka
    restart: unless-stopped

  # StatsD Exporter для Airflow метрик (StatsD → Prometheus)
  statsd-exporter:
    image: prom/statsd-exporter:v0.27.1
    command: [
      "--statsd.listen-udp=:8125",
      "--statsd.listen-tcp=",
      "--web.listen-address=:9102",
      "--statsd.mapping-config=/tmp/statsd_mapping.yml"
    ]
    ports:
      - "9102:9102"     # Prometheus metrics endpoint
      - "8125:8125/udp" # StatsD receive port
    volumes:
      - ./configs/statsd_mapping.yml:/tmp/statsd_mapping.yml:ro
    networks:
      - cs_dwh
    restart: unless-stopped

networks:
  cs_dwh:
    driver: bridge

volumes:
  clickhouse-data:
  grafana_lib:
  kafka-data:
  pgmeta:
  superset_data:
  superset_config: