name: ${COMPOSE_PROJECT_NAME:-clickstream-data-platform} x-clickhouse-nofile: &clickhouse-nofile nofile: soft: 262144 hard: 262144 x-clickhouse-common: &clickhouse-common image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56} restart: unless-stopped environment: # Стенд локальный: нодам нужен общий пользователь без пароля для Distributed. CLICKHOUSE_SKIP_USER_SETUP: "1" depends_on: clickhouse-keeper: condition: service_healthy ulimits: *clickhouse-nofile # Гигабайта не хватало: половину съедали страницы самого бинарника (ADR 0004). mem_limit: 4g healthcheck: test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --query 'SELECT 1' >/dev/null 2>&1"] interval: 5s timeout: 3s retries: 30 start_period: 10s x-airflow-common: &airflow-common image: clickstream-airflow:local build: context: ./infra/airflow args: AIRFLOW_BASE_IMAGE: ${AIRFLOW_IMAGE:-apache/airflow:3.3.0} environment: AIRFLOW__CORE__EXECUTOR: LocalExecutor AIRFLOW__CORE__PARALLELISM: "4" AIRFLOW__CORE__LOAD_EXAMPLES: "false" AIRFLOW__CORE__AUTH_MANAGER: airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin" AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/auth/simple_auth_manager_passwords.json AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-apiserver:8080/execution/ AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=} AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_JWT_SECRET:-local-only-airflow-jwt-secret-do-not-use} AIRFLOW__API_AUTH__JWT_ISSUER: airflow AIRFLOW__API__SECRET_KEY: ${AIRFLOW_API_SECRET_KEY:-local-only-airflow-api-secret-do-not-use} AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: "postgresql+psycopg2://${AIRFLOW_METADATA_USER:-airflow}:${AIRFLOW_METADATA_PASSWORD:-airflow-local}@postgres-metadata:5432/airflow" AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true" AIRFLOW_ADMIN_USER: ${AIRFLOW_ADMIN_USER:-admin} AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-airflow} volumes: - ./dags:/opt/airflow/dags:ro - ./infra/airflow/init.sh:/opt/airflow/init.sh:ro - airflow_logs:/opt/airflow/logs - airflow_auth:/opt/airflow/auth x-generator-common: &generator-common image: clickstream-generator:local build: context: . dockerfile: generator/Dockerfile restart: "no" depends_on: kafka-init: condition: service_completed_successfully # Читать топик движок Kafka начинает не при своём создании, а когда над ним # появляется матвью приёма, и она создаётся последней по номеру файла # (sql/ddl/40-stg-views.sql). Отправь генератор события раньше — они лягут в # топик и молча минуют хранилище. clickhouse-init: condition: service_completed_successfully environment: # Адрес брокера и имя топика — факты стенда, и называет их стенд. # Остальное (день, зерно, число дней, предел пачки) приходит аргументами # от того, кто запускает: у службы нет позиции на оси мира. KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} KAFKA_TOPIC: ${KAFKA_TOPIC:-hits} x-superset-common: &superset-common image: clickstream-superset:local build: context: ./infra/superset args: SUPERSET_BASE_IMAGE: ${SUPERSET_IMAGE:-apache/superset:6.1.0} environment: SUPERSET_CONFIG_PATH: /app/pythonpath/superset_config.py SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY:-local-only-superset-secret-do-not-use-outside-this-stand} SUPERSET__SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://${SUPERSET_METADATA_USER:-superset}:${SUPERSET_METADATA_PASSWORD:-superset-local}@postgres-metadata:5432/superset" SUPERSET_ADMIN_USER: ${SUPERSET_ADMIN_USER:-admin} SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD:-superset} volumes: - ./infra/superset/superset_config.py:/app/pythonpath/superset_config.py:ro - ./infra/superset/init.sh:/app/infra/init.sh:ro - ./infra/superset/import:/app/import:ro - superset_home:/app/superset_home services: clickhouse-keeper: image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56} restart: unless-stopped user: clickhouse ulimits: *clickhouse-nofile entrypoint: - clickhouse - keeper - --config-file=/etc/clickhouse-keeper/keeper_config.xml volumes: - ./infra/clickhouse/keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml:ro - clickhouse_keeper_data:/var/lib/clickhouse mem_limit: 512m healthcheck: test: ["CMD-SHELL", "clickhouse keeper-client --host 127.0.0.1 --port 9181 --query ruok | grep -q imok"] interval: 5s timeout: 3s retries: 30 start_period: 10s # Инициатор DDL и будущее подключение Airflow. clickhouse-01: <<: *clickhouse-common hostname: clickhouse-01 ports: - "127.0.0.1:${CLICKHOUSE_01_HTTP_PORT:-28123}:8123" - "127.0.0.1:${CLICKHOUSE_01_TCP_PORT:-29000}:9000" volumes: - ./infra/clickhouse/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro - ./infra/clickhouse/config.d/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro - ./infra/clickhouse/config.d/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml:ro - clickhouse_01_data:/var/lib/clickhouse # Будущее подключение Superset. clickhouse-02: <<: *clickhouse-common hostname: clickhouse-02 ports: - "127.0.0.1:${CLICKHOUSE_02_HTTP_PORT:-28124}:8123" - "127.0.0.1:${CLICKHOUSE_02_TCP_PORT:-29001}:9000" volumes: - ./infra/clickhouse/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro - ./infra/clickhouse/config.d/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro - ./infra/clickhouse/config.d/macros-02.xml:/etc/clickhouse-server/config.d/macros.xml:ro - clickhouse_02_data:/var/lib/clickhouse kafka: image: ${KAFKA_IMAGE:-apache/kafka:4.3.1} hostname: kafka restart: unless-stopped ports: - "127.0.0.1:${KAFKA_EXTERNAL_PORT:-29092}:29092" environment: KAFKA_NODE_ID: "1" KAFKA_PROCESS_ROLES: "broker,controller" KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093" KAFKA_LISTENERS: "INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:29092" KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:9092,EXTERNAL://localhost:${KAFKA_EXTERNAL_PORT:-29092}" KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT" KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1" KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1" KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: "1" KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: "1" KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: "0" KAFKA_NUM_PARTITIONS: "1" KAFKA_LOG_DIRS: "/var/lib/kafka/data" KAFKA_HEAP_OPTS: "-Xmx512m -Xms256m" CLUSTER_ID: "4L6g3nShT-eMCtK--X86sw" volumes: - kafka_data:/var/lib/kafka/data mem_limit: 1g healthcheck: test: ["CMD-SHELL", "/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list >/dev/null 2>&1"] interval: 15s timeout: 10s retries: 20 start_period: 30s kafka-init: image: ${KAFKA_IMAGE:-apache/kafka:4.3.1} restart: "no" depends_on: kafka: condition: service_healthy entrypoint: ["/bin/bash", "-ec"] command: - | topic_partition_count() { /opt/kafka/bin/kafka-topics.sh \ --bootstrap-server kafka:9092 \ --describe \ --topic hits | \ sed -n '1s/.*PartitionCount: \([0-9][0-9]*\).*/\1/p' } /opt/kafka/bin/kafka-topics.sh \ --bootstrap-server kafka:9092 \ --create \ --if-not-exists \ --topic hits \ --partitions 2 \ --replication-factor 1 partition_count="$$(topic_partition_count)" # Брокер разрешает автосоздание, хотя потребитель ClickHouse его не запрашивает. # На сохранённом томе другой клиент мог уже создать hits с одним разделом. if [[ "$$partition_count" == "1" ]]; then /opt/kafka/bin/kafka-topics.sh \ --bootstrap-server kafka:9092 \ --alter \ --topic hits \ --partitions 2 partition_count="$$(topic_partition_count)" fi if [[ "$$partition_count" != "2" ]]; then printf 'ОШИБКА: у топика hits разделов: %s, ожидалось: 2.\n' \ "$${partition_count:-неизвестно}" >&2 exit 1 fi # DDL применяется с ноды 1 по порядку имён файлов после готовности всего кластера. clickhouse-init: image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56} restart: "no" depends_on: kafka-init: condition: service_completed_successfully clickhouse-01: condition: service_healthy clickhouse-02: condition: service_healthy environment: LC_ALL: C entrypoint: ["/bin/bash", "-ec"] command: - | set -- /ddl/*.sql if [[ ! -e "$$1" ]]; then printf 'ОШИБКА: в /ddl нет файлов SQL.\n' >&2 exit 1 fi for ddl_file do printf 'Применяем %s.\n' "$${ddl_file##*/}" clickhouse-client \ --host clickhouse-01 \ --multiquery \ --queries-file "$$ddl_file" done volumes: - ./sql/ddl:/ddl:ro # Стартовый мир: восемь модельных дней уезжают в топик при каждом подъёме. # Что это за дни и каким мир обязан выйти — data/world-inventory.json. # # Число дней стоит здесь числом: YAML не читает Python, и одно из двух мест # (второе — `STARTING_DAYS` в inventory.py) лишнее по построению. Разъедутся # они — покраснеют счётчики `make check-clickhouse`. # # Повторный `make up` заливает мир заново, и это не оплошность: `WatchID` у # событий те же, ReplacingMergeTree схлопнет повтор в ODS. Сырьё в STG при # этом честно удвоится — свойство слоя, описанное в storage.md. world-init: <<: *generator-common command: ["batch", "--day", "0", "--days", "8"] # Тот же образ для ручных прогонов: `make generate-batch`, `make generate-live`. # Под профилем — чтобы обычный подъём стенда её не трогал. generator: <<: *generator-common profiles: ["generator"] command: ["batch"] postgres-metadata: image: ${POSTGRES_IMAGE:-postgres:16-alpine} restart: unless-stopped command: - postgres - -c - shared_buffers=64MB - -c - max_connections=50 environment: POSTGRES_USER: ${POSTGRES_ADMIN_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD:-postgres-local} POSTGRES_DB: postgres AIRFLOW_METADATA_USER: ${AIRFLOW_METADATA_USER:-airflow} AIRFLOW_METADATA_PASSWORD: ${AIRFLOW_METADATA_PASSWORD:-airflow-local} SUPERSET_METADATA_USER: ${SUPERSET_METADATA_USER:-superset} SUPERSET_METADATA_PASSWORD: ${SUPERSET_METADATA_PASSWORD:-superset-local} volumes: - ./infra/postgres/init/10-metadata-databases.sh:/docker-entrypoint-initdb.d/10-metadata-databases.sh:ro - postgres_metadata_data:/var/lib/postgresql/data mem_limit: 256m healthcheck: test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d postgres >/dev/null"] interval: 5s timeout: 5s retries: 20 start_period: 10s airflow-init: <<: *airflow-common user: "0:0" restart: "no" depends_on: postgres-metadata: condition: service_healthy clickhouse-01: condition: service_healthy # Не убирать: без зависимого успешный clickhouse-init считается # упавшим для --wait. clickhouse-init: condition: service_completed_successfully # По той же причине — и ещё по одной. Заливка стартового мира идёт # последней в цепи разовых служб, и зависимого ей взять больше негде. # Заодно это правда про порядок: даг `next_day` этапа 5 продолжает ось с # того дня, на котором заливка остановилась, — Airflow приходит в мир, # который уже есть. world-init: condition: service_completed_successfully entrypoint: ["/bin/bash"] command: ["/opt/airflow/init.sh"] airflow-apiserver: <<: *airflow-common restart: unless-stopped depends_on: airflow-init: condition: service_completed_successfully command: api-server ports: - "127.0.0.1:${AIRFLOW_PORT:-28080}:8080" mem_limit: 512m healthcheck: test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8080/api/v2/monitor/health | python -c 'import json,sys; sys.exit(0 if json.load(sys.stdin)[\"metadatabase\"][\"status\"] == \"healthy\" else 1)'"] interval: 10s timeout: 5s retries: 30 start_period: 30s airflow-scheduler: <<: *airflow-common restart: unless-stopped depends_on: airflow-init: condition: service_completed_successfully command: scheduler mem_limit: 640m healthcheck: test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8974/health >/dev/null"] interval: 10s timeout: 5s retries: 30 start_period: 30s airflow-dag-processor: <<: *airflow-common restart: unless-stopped depends_on: airflow-init: condition: service_completed_successfully airflow-apiserver: condition: service_healthy command: dag-processor mem_limit: 384m healthcheck: test: ["CMD-SHELL", "curl -sf http://airflow-apiserver:8080/api/v2/monitor/health | python -c 'import json,sys; sys.exit(0 if json.load(sys.stdin)[\"dag_processor\"][\"status\"] == \"healthy\" else 1)'"] interval: 10s timeout: 5s retries: 30 start_period: 30s superset-init: <<: *superset-common restart: "no" healthcheck: disable: true depends_on: postgres-metadata: condition: service_healthy clickhouse-02: condition: service_healthy entrypoint: ["/bin/bash"] command: ["/app/infra/init.sh"] superset: <<: *superset-common restart: unless-stopped depends_on: superset-init: condition: service_completed_successfully ports: - "127.0.0.1:${SUPERSET_PORT:-28088}:8088" mem_limit: 1g healthcheck: test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8088/health >/dev/null"] interval: 10s timeout: 5s retries: 30 start_period: 30s prometheus: image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.13.2} restart: unless-stopped depends_on: clickhouse-keeper: condition: service_healthy clickhouse-01: condition: service_healthy clickhouse-02: condition: service_healthy command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=7d - --storage.tsdb.retention.size=512MB volumes: - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus_data:/prometheus ports: - "127.0.0.1:${PROMETHEUS_PORT:-29090}:9090" mem_limit: 512m healthcheck: test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9090/-/ready >/dev/null 2>&1"] interval: 10s timeout: 5s retries: 15 start_period: 10s grafana: image: ${GRAFANA_IMAGE:-grafana/grafana:13.1.1} restart: unless-stopped depends_on: prometheus: condition: service_healthy environment: GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} GF_USERS_ALLOW_SIGN_UP: "false" GF_NEWS_NEWS_FEED_ENABLED: "false" GF_ANALYTICS_REPORTING_ENABLED: "false" GF_ANALYTICS_CHECK_FOR_UPDATES: "false" volumes: - ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro - grafana_data:/var/lib/grafana ports: - "127.0.0.1:${GRAFANA_PORT:-23000}:3000" mem_limit: 512m healthcheck: test: ["CMD-SHELL", "curl -sf http://127.0.0.1:3000/api/health >/dev/null"] interval: 10s timeout: 5s retries: 15 start_period: 15s volumes: clickhouse_keeper_data: clickhouse_01_data: clickhouse_02_data: kafka_data: postgres_metadata_data: airflow_logs: airflow_auth: superset_home: prometheus_data: grafana_data: