clickstream-ch-kafka-superset-demo/docker-compose.yml

x-airflow-env: &airflow-default-env
  AIRFLOW__CORE__LOAD_EXAMPLES: "False"
  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow
  AIRFLOW__CORE__EXECUTOR: LocalExecutor
  AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW_SECRET_KEY:-replace-me-with-random-string}
  # ClickHouse connection для ETL
  AIRFLOW_CONN_CLICKHOUSE_DEFAULT: clickhouse://default:123456@clickhouse:9000/default
  # StatsD для мониторинга (метрики → statsd-exporter → Prometheus)
  AIRFLOW__METRICS__STATSD_ON: "True"
  AIRFLOW__METRICS__STATSD_HOST: "statsd-exporter"
  AIRFLOW__METRICS__STATSD_PORT: "8125"
  AIRFLOW__METRICS__STATSD_PREFIX: "airflow"
  PYTHONPATH: /opt/airflow/generator_src


services:
  clickhouse:
    image: clickhouse/clickhouse-server:25.1
    environment:
        TZ: "Europe/Moscow"
    ulimits:
      nproc: 65535
      nofile:
        soft: 262144
        hard: 262144
    networks:
      - cs_dwh
    volumes:
      - clickhouse-data:/var/lib/clickhouse
      - ./configs/default_user.xml:/etc/clickhouse-server/users.d/default_user.xml
      # - ./configs/z_config.xml:/etc/clickhouse-server/config.d/z_config.xml
      # - ./configs/macros_ch1.xml:/etc/clickhouse-server/config.d/macros.xml
      # - ./configs/clickhouse-backup.yml:/etc/clickhouse-backup/config.yml
      # - ./configs/s3-storage.xml:/etc/clickhouse-server/config.d/s3-storage.xml
      - ./configs/prometheus_ch.xml:/etc/clickhouse-server/config.d/s3-prometheus_ch.xml
    ports:
      - "8002:9000"
      - "9123:8123"
      - "9126:9126"  # ClickHouse Prometheus metrics endpoint

  kafka:
    image: apache/kafka:3.8.0
    # hostname: kafka
    # container_name: kafka
    ports:
      - 9092:9092
    # Данные топиков/сообщений сохраняются в named volume `kafka-data` (между `docker compose down/up`).
    volumes:
      - kafka-data:/var/lib/kafka/data
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
      KAFKA_LISTENERS: PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
    networks:
      - cs_dwh

  kafka-ui:
    # container_name: kafka-ui
    image: provectuslabs/kafka-ui:v0.7.2
    ports:
      - 8082:8080
    depends_on:
      - kafka
    environment:
      DYNAMIC_CONFIG_ENABLED: "true"
      KAFKA_CLUSTERS_0_NAME: "Kafka Cluster"
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka:29092"
      KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: "PLAINTEXT"
    networks:
      - cs_dwh

  # PostgreSQL for Airflow metadata
  postgres-metadata:
    image: postgres:16
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    ports:
      - "5434:5432"
    volumes:
      - pgmeta:/var/lib/postgresql/data
    networks:
      - cs_dwh
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U airflow -d airflow"]
      interval: 5s
      timeout: 5s
      retries: 20

  # Airflow webserver
  airflow-webserver:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    image: airflow-optimized:2.10.5
    environment:
      <<: *airflow-default-env
    command: >
      bash -c "
        airflow webserver
      "
    ports:
      - "8080:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./generator/src:/opt/airflow/generator_src:ro
      - ./sql:/opt/airflow/sql:ro
      - ./data:/opt/airflow/data
    networks:
      - cs_dwh
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres-metadata:
        condition: service_healthy
      clickhouse:
        condition: service_started

  # Airflow scheduler
  airflow-scheduler:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    environment:
      <<: *airflow-default-env
    command: >
      bash -c "
        airflow scheduler
      "
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./generator/src:/opt/airflow/generator_src:ro
      - ./sql:/opt/airflow/sql:ro
      - ./data:/opt/airflow/data
    networks:
      - cs_dwh
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres-metadata:
        condition: service_healthy
      clickhouse:
        condition: service_started

  # Airflow init to create admin user
  airflow-init:
    build:
      context: .
      dockerfile: Dockerfile.airflow
    user: "0:0"
    environment:
      <<: *airflow-default-env
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./generator/src:/opt/airflow/generator_src:ro
      - ./sql:/opt/airflow/sql:ro
      - ./data:/opt/airflow/data
    networks:
      - cs_dwh
    command: >
      bash -ceuo pipefail "
        mkdir -p /opt/airflow/data &&
        chmod -R a+rX /opt/airflow/data || true &&
        chown -R airflow:0 /opt/airflow/data || true &&
        umask 000 &&
        su -s /bin/bash airflow -c 'airflow db migrate' &&
        su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' &&
        su -s /bin/bash airflow -c 'airflow connections delete clickhouse_default || true' &&
        su -s /bin/bash airflow -c \"airflow connections add 'clickhouse_default' --conn-uri \\\"$${AIRFLOW_CONN_CLICKHOUSE_DEFAULT}\\\" --conn-description 'ClickHouse DWH'\"
      "
    depends_on:
      postgres-metadata:
        condition: service_healthy
      clickhouse:
        condition: service_started

  prometheus:
    image: prom/prometheus:v2.53.4
    volumes:
      - ./configs/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    # container_name: prometheus
    # hostname: prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
    ports:
      - 9090:9090
    restart: unless-stopped
    environment:
      TZ: "Europe/Moscow"
    networks:
      - cs_dwh

  grafana:
    image: grafana/grafana:11.5.2-ubuntu
    depends_on:
      - prometheus
    ports:
      - 3000:3000
    volumes:
      - grafana_lib:/var/lib/grafana
      - ./configs/grafana/provisioning/:/etc/grafana/provisioning/:ro
    # container_name: grafana
    # hostname: grafana
    restart: unless-stopped
    environment:
      TZ: "Europe/Moscow"
    networks:
      - cs_dwh

  superset:
    # image: apache/superset
    build:
      context: .
      dockerfile: Dockerfile.superset
    restart: unless-stopped
    ports:
      - "8088:8088"
    networks:
      - cs_dwh
    volumes:
      - superset_data:/var/lib/superset
      - superset_config:/app/superset_home
      - ./superset:/app/superset_init:ro
      - ./configs/superset_config.py:/app/pythonpath/superset_config.py:ro
    environment:
      - SUPERSET_SECRET_KEY=9wc5+erMt60+lxrXDf3RjeIR+zONpEFusO00Np7JzfliMTI1e+RXnHcQ
      - TZ=Europe/Moscow
      - SUPERSET_CONFIG_PATH=/app/pythonpath/superset_config.py
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
      interval: 30s
      timeout: 10s
      retries: 5
    depends_on:
      superset-init:
        condition: service_completed_successfully

  # Superset initialization service
  superset-init:
    build:
      context: .
      dockerfile: Dockerfile.superset
    user: "0:0"
    networks:
      - cs_dwh
    volumes:
      - superset_data:/var/lib/superset
      - superset_config:/app/superset_home
      - ./superset:/app/superset_init:ro
      - ./configs/superset_config.py:/app/pythonpath/superset_config.py:ro
    environment:
      - SUPERSET_SECRET_KEY=9wc5+erMt60+lxrXDf3RjeIR+zONpEFusO00Np7JzfliMTI1e+RXnHcQ
      - TZ=Europe/Moscow
      - SUPERSET_CONFIG_PATH=/app/pythonpath/superset_config.py
    command: >
      bash -ceuo pipefail "
        echo 'Waiting for PostgreSQL...' &&
        sleep 5 &&
        echo 'Creating superset database if not exists...' &&
        PGPASSWORD=airflow psql -h postgres-metadata -U airflow -d airflow -tc \"SELECT 1 FROM pg_database WHERE datname='superset'\" | grep -q 1 || \\
          PGPASSWORD=airflow psql -h postgres-metadata -U airflow -d airflow -c \"CREATE DATABASE superset;\" &&
        echo 'Initializing Superset DB...' &&
        superset db upgrade &&
        echo 'Creating admin user...' &&
        superset fab create-admin --username admin --password admin --firstname Superset --lastname Admin --email admin@example.org || true &&
        echo 'Initializing roles...' &&
        superset init &&
        echo 'Creating ClickHouse connection and datasets...' &&
        python /app/superset_init/init_superset.py &&
        echo 'Creating dashboard and charts...' &&
        python /app/superset_init/create_dashboard.py &&
        echo 'Superset initialized successfully'
      "
    depends_on:
      postgres-metadata:
        condition: service_healthy

  # Генератор событий (автономный стриминг в Kafka)
  generator:
    profiles:
      - live-generator
    build:
      context: ./generator
      dockerfile: Dockerfile
    environment:
      KAFKA_BOOTSTRAP_SERVERS: kafka:29092
      # Режим steady-stream: короткие тики 1-10 сек (по умолчанию 5)
      GEN_TICK_SECONDS: ${GEN_TICK_SECONDS:-5}
      GEN_LAMBDA_BASE_PER_MIN: ${GEN_LAMBDA_BASE_PER_MIN:-30}
      GEN_JITTER_PCT: ${GEN_JITTER_PCT:-20}
      GEN_MIN_EVENTS_PER_TICK: ${GEN_MIN_EVENTS_PER_TICK:-1}
      GEN_MAX_EVENTS_PER_TICK: ${GEN_MAX_EVENTS_PER_TICK:-50}
      GEN_MAX_SESSION_EVENTS: ${GEN_MAX_SESSION_EVENTS:-30}
      GEN_MAX_ACTIVE_SESSIONS: ${GEN_MAX_ACTIVE_SESSIONS:-200}
      GEN_POPULATION_MAX: ${GEN_POPULATION_MAX:-300}
      GEN_P_NEW_USER: ${GEN_P_NEW_USER:-0.15}
      GEN_MIN_RETURN_MINUTES: ${GEN_MIN_RETURN_MINUTES:-30}
      GEN_MODEL_T0: ${GEN_MODEL_T0:-2026-01-01T00:00:00+00:00}
      GEN_MODEL_T_END: ${GEN_MODEL_T_END:-}
      GEN_MODEL_TIMEZONE: ${GEN_MODEL_TIMEZONE:-UTC}
      GEN_MODEL_TIME_SPEED: ${GEN_MODEL_TIME_SPEED:-1}
      GEN_RUN_MODE: ${GEN_RUN_MODE:-live}
      GEN_LAUNCH_PROFILE: ${GEN_LAUNCH_PROFILE:-ci}
      GEN_DATA_DIR: /data
      GEN_SEED: ${GEN_SEED:-}
      GEN_ENABLED: ${GEN_ENABLED:-true}
      GEN_METRICS_PORT: ${GEN_METRICS_PORT:-9109}
      # Сохранение состояния между рестартами
      GEN_STATE_ENABLED: ${GEN_STATE_ENABLED:-true}
      GEN_STATE_RESET: ${GEN_STATE_RESET:-false}
      # PYTHONUNBUFFERED для сразу видеть логи
      PYTHONUNBUFFERED: "1"
    ports:
      - "9109:9109"  # Prometheus metrics endpoint
    volumes:
      - ./data:/data:ro
    networks:
      - cs_dwh
    depends_on:
      - kafka
    restart: unless-stopped
    # SIGTERM останавливает генератор на границе пакета; минута нужна на сброс буфера Kafka.
    stop_grace_period: 1m
    healthcheck:
      test: ["CMD", "python", "-c", "import sys; sys.exit(0)"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Kafka Exporter для мониторинга через Prometheus
  kafka-exporter:
    image: danielqsj/kafka-exporter:v1.9.0
    command: ["--kafka.server=kafka:29092"]
    ports:
      - "9308:9308"
    networks:
      - cs_dwh
    depends_on:
      - kafka
    restart: unless-stopped

  # StatsD Exporter для Airflow метрик (StatsD → Prometheus)
  statsd-exporter:
    image: prom/statsd-exporter:v0.27.1
    command: [
      "--statsd.listen-udp=:8125",
      "--statsd.listen-tcp=",
      "--web.listen-address=:9102",
      "--statsd.mapping-config=/tmp/statsd_mapping.yml"
    ]
    ports:
      - "9102:9102"     # Prometheus metrics endpoint
      - "8125:8125/udp" # StatsD receive port
    volumes:
      - ./configs/statsd_mapping.yml:/tmp/statsd_mapping.yml:ro
    networks:
      - cs_dwh
    restart: unless-stopped

networks:
  cs_dwh:
    driver: bridge

volumes:
  clickhouse-data:
  grafana_lib:
  kafka-data:
  pgmeta:
  superset_data:
  superset_config: