ddmitry/clickstream-data-platform
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects 1 Releases Wiki Activity
Files
812b398ce2f7ff80cf3faab113eff5dd63129578
clickstream-data-platform/compose.yaml
T
ddadminandClaude Opus 5 64f3418378 feat(airflow): пробные DAG test_clickhouse и test_kafka вместо демонстрационного 
Зачем.
Демонстрационный DAG example_clickstream_hello ничего не проверял: он не
обращался ни к ClickHouse, ни к Kafka, поэтому его зелёный результат ничего
не говорил о стенде. Пробники проверяют связи по-настоящему — и тем же
клиентом, каким будут ходить рабочие DAG.

Что.
- test_clickhouse: пишет строку в ReplicatedMergeTree на ноде 1 и читает её
  с ноды 2 через Distributed. Данные проходят путь «нода 2 → все шарды →
  шард ноды 1», то есть проверяется межшардовое чтение, а не одна нода.
- test_kafka: пишет в постоянный топик сообщение с меткой прогона и
  вычитывает его обратно.
- infra/airflow/Dockerfile: clickhouse-connect 1.6.0 и confluent-kafka
  2.15.0 вшиты в образ, импорт проверяется на сборке — при запуске
  контейнера пакеты не доустанавливаются.
- Проверки: scripts/stand-smoke.sh гоняет оба пробника через API Airflow,
  scripts/config-test.sh разбирает DAG без стенда,
  tests/stand-smoke-guards.sh проверяет красный путь,
  tests/dag-probes-unit.py — модульные проверки разбора.
- README и ADR 0001 обновлены тем же изменением.
- Удалён dags/example_clickstream_hello.py.

Проверка.
make config-test — пройдено 3, 3 и 6, ошибок 0.
make clean; cp .env.example .env; make up — 116 с на чистых томах.
make smoke — пройдено 25, ошибок 0; стенд занимает 2244,0 MiB.
make smoke-cluster — все 8 проверок кластера.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 14:04:19 +03:00

335 lines
12 KiB
YAML
Raw Blame History

name: ${COMPOSE_PROJECT_NAME:-clickstream-data-platform}
x-clickhouse-nofile: &clickhouse-nofile
nofile:
soft: 262144
hard: 262144
x-clickhouse-common: &clickhouse-common
image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56}
restart: unless-stopped
environment:
# Стенд локальный: нодам нужен общий пользователь без пароля для Distributed.
CLICKHOUSE_SKIP_USER_SETUP: "1"
depends_on:
clickhouse-keeper:
condition: service_healthy
ulimits: *clickhouse-nofile
mem_limit: 1g
healthcheck:
test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --query 'SELECT 1' >/dev/null 2>&1"]
interval: 5s
timeout: 3s
retries: 30
start_period: 10s
x-airflow-common: &airflow-common
image: clickstream-airflow:local
build:
context: ./infra/airflow
args:
AIRFLOW_BASE_IMAGE: ${AIRFLOW_IMAGE:-apache/airflow:3.3.0}
environment:
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__CORE__PARALLELISM: "4"
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
AIRFLOW__CORE__AUTH_MANAGER: airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/auth/simple_auth_manager_passwords.json
AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-apiserver:8080/execution/
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=}
AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_JWT_SECRET:-local-only-airflow-jwt-secret-do-not-use}
AIRFLOW__API_AUTH__JWT_ISSUER: airflow
AIRFLOW__API__SECRET_KEY: ${AIRFLOW_API_SECRET_KEY:-local-only-airflow-api-secret-do-not-use}
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: "postgresql+psycopg2://${AIRFLOW_METADATA_USER:-airflow}:${AIRFLOW_METADATA_PASSWORD:-airflow-local}@postgres-metadata:5432/airflow"
AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true"
AIRFLOW_ADMIN_USER: ${AIRFLOW_ADMIN_USER:-admin}
AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-airflow}
volumes:
- ./dags:/opt/airflow/dags:ro
- ./infra/airflow/init.sh:/opt/airflow/init.sh:ro
- airflow_logs:/opt/airflow/logs
- airflow_auth:/opt/airflow/auth
x-superset-common: &superset-common
image: clickstream-superset:local
build:
context: ./infra/superset
args:
SUPERSET_BASE_IMAGE: ${SUPERSET_IMAGE:-apache/superset:6.1.0}
environment:
SUPERSET_CONFIG_PATH: /app/pythonpath/superset_config.py
SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY:-local-only-superset-secret-do-not-use-outside-this-stand}
SUPERSET__SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://${SUPERSET_METADATA_USER:-superset}:${SUPERSET_METADATA_PASSWORD:-superset-local}@postgres-metadata:5432/superset"
SUPERSET_ADMIN_USER: ${SUPERSET_ADMIN_USER:-admin}
SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD:-superset}
volumes:
- ./infra/superset/superset_config.py:/app/pythonpath/superset_config.py:ro
- ./infra/superset/init.sh:/app/infra/init.sh:ro
- ./infra/superset/import:/app/import:ro
- superset_home:/app/superset_home
services:
clickhouse-keeper:
image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56}
restart: unless-stopped
user: clickhouse
ulimits: *clickhouse-nofile
entrypoint:
- clickhouse
- keeper
- --config-file=/etc/clickhouse-keeper/keeper_config.xml
volumes:
- ./infra/clickhouse/keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml:ro
- clickhouse_keeper_data:/var/lib/clickhouse
mem_limit: 512m
healthcheck:
test: ["CMD-SHELL", "clickhouse keeper-client --host 127.0.0.1 --port 9181 --query ruok | grep -q imok"]
interval: 5s
timeout: 3s
retries: 30
start_period: 10s
# Инициатор DDL и будущее подключение Airflow.
clickhouse-01:
<<: *clickhouse-common
ports:
- "127.0.0.1:${CLICKHOUSE_01_HTTP_PORT:-28123}:8123"
- "127.0.0.1:${CLICKHOUSE_01_TCP_PORT:-29000}:9000"
volumes:
- ./infra/clickhouse/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
- ./infra/clickhouse/config.d/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
- ./infra/clickhouse/config.d/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml:ro
- clickhouse_01_data:/var/lib/clickhouse
# Будущее подключение Superset.
clickhouse-02:
<<: *clickhouse-common
ports:
- "127.0.0.1:${CLICKHOUSE_02_HTTP_PORT:-28124}:8123"
- "127.0.0.1:${CLICKHOUSE_02_TCP_PORT:-29001}:9000"
volumes:
- ./infra/clickhouse/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
- ./infra/clickhouse/config.d/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
- ./infra/clickhouse/config.d/macros-02.xml:/etc/clickhouse-server/config.d/macros.xml:ro
- clickhouse_02_data:/var/lib/clickhouse
kafka:
image: ${KAFKA_IMAGE:-apache/kafka:4.3.1}
hostname: kafka
restart: unless-stopped
ports:
- "127.0.0.1:${KAFKA_EXTERNAL_PORT:-29092}:29092"
environment:
KAFKA_NODE_ID: "1"
KAFKA_PROCESS_ROLES: "broker,controller"
KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
KAFKA_LISTENERS: "INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:29092"
KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:9092,EXTERNAL://localhost:${KAFKA_EXTERNAL_PORT:-29092}"
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT"
KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1"
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: "1"
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: "1"
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: "0"
KAFKA_NUM_PARTITIONS: "1"
KAFKA_LOG_DIRS: "/var/lib/kafka/data"
KAFKA_HEAP_OPTS: "-Xmx512m -Xms256m"
CLUSTER_ID: "4L6g3nShT-eMCtK--X86sw"
volumes:
- kafka_data:/var/lib/kafka/data
mem_limit: 1g
healthcheck:
test: ["CMD-SHELL", "/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list >/dev/null 2>&1"]
interval: 15s
timeout: 10s
retries: 20
start_period: 30s
postgres-metadata:
image: ${POSTGRES_IMAGE:-postgres:16-alpine}
restart: unless-stopped
command:
- postgres
- -c
- shared_buffers=64MB
- -c
- max_connections=50
environment:
POSTGRES_USER: ${POSTGRES_ADMIN_USER:-postgres}
POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD:-postgres-local}
POSTGRES_DB: postgres
AIRFLOW_METADATA_USER: ${AIRFLOW_METADATA_USER:-airflow}
AIRFLOW_METADATA_PASSWORD: ${AIRFLOW_METADATA_PASSWORD:-airflow-local}
SUPERSET_METADATA_USER: ${SUPERSET_METADATA_USER:-superset}
SUPERSET_METADATA_PASSWORD: ${SUPERSET_METADATA_PASSWORD:-superset-local}
volumes:
- ./infra/postgres/init/10-metadata-databases.sh:/docker-entrypoint-initdb.d/10-metadata-databases.sh:ro
- postgres_metadata_data:/var/lib/postgresql/data
mem_limit: 256m
healthcheck:
test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d postgres >/dev/null"]
interval: 5s
timeout: 5s
retries: 20
start_period: 10s
airflow-init:
<<: *airflow-common
user: "0:0"
restart: "no"
depends_on:
postgres-metadata:
condition: service_healthy
clickhouse-01:
condition: service_healthy
entrypoint: ["/bin/bash"]
command: ["/opt/airflow/init.sh"]
airflow-apiserver:
<<: *airflow-common
restart: unless-stopped
depends_on:
airflow-init:
condition: service_completed_successfully
command: api-server
ports:
- "127.0.0.1:${AIRFLOW_PORT:-28080}:8080"
mem_limit: 512m
healthcheck:
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8080/api/v2/monitor/health | python -c 'import json,sys; sys.exit(0 if json.load(sys.stdin)[\"metadatabase\"][\"status\"] == \"healthy\" else 1)'"]
interval: 10s
timeout: 5s
retries: 30
start_period: 30s
airflow-scheduler:
<<: *airflow-common
restart: unless-stopped
depends_on:
airflow-init:
condition: service_completed_successfully
command: scheduler
mem_limit: 640m
healthcheck:
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8974/health >/dev/null"]
interval: 10s
timeout: 5s
retries: 30
start_period: 30s
airflow-dag-processor:
<<: *airflow-common
restart: unless-stopped
depends_on:
airflow-init:
condition: service_completed_successfully
airflow-apiserver:
condition: service_healthy
command: dag-processor
mem_limit: 384m
healthcheck:
test: ["CMD-SHELL", "curl -sf http://airflow-apiserver:8080/api/v2/monitor/health | python -c 'import json,sys; sys.exit(0 if json.load(sys.stdin)[\"dag_processor\"][\"status\"] == \"healthy\" else 1)'"]
interval: 10s
timeout: 5s
retries: 30
start_period: 30s
superset-init:
<<: *superset-common
restart: "no"
healthcheck:
disable: true
depends_on:
postgres-metadata:
condition: service_healthy
clickhouse-02:
condition: service_healthy
entrypoint: ["/bin/bash"]
command: ["/app/infra/init.sh"]
superset:
<<: *superset-common
restart: unless-stopped
depends_on:
superset-init:
condition: service_completed_successfully
ports:
- "127.0.0.1:${SUPERSET_PORT:-28088}:8088"
mem_limit: 1g
healthcheck:
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8088/health >/dev/null"]
interval: 10s
timeout: 5s
retries: 30
start_period: 30s
prometheus:
image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.13.2}
restart: unless-stopped
depends_on:
clickhouse-keeper:
condition: service_healthy
clickhouse-01:
condition: service_healthy
clickhouse-02:
condition: service_healthy
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=512MB
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
ports:
- "127.0.0.1:${PROMETHEUS_PORT:-29090}:9090"
mem_limit: 512m
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9090/-/ready >/dev/null 2>&1"]
interval: 10s
timeout: 5s
retries: 15
start_period: 10s
grafana:
image: ${GRAFANA_IMAGE:-grafana/grafana:13.1.1}
restart: unless-stopped
depends_on:
prometheus:
condition: service_healthy
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
GF_USERS_ALLOW_SIGN_UP: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
volumes:
- ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
- grafana_data:/var/lib/grafana
ports:
- "127.0.0.1:${GRAFANA_PORT:-23000}:3000"
mem_limit: 512m
healthcheck:
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:3000/api/health >/dev/null"]
interval: 10s
timeout: 5s
retries: 15
start_period: 15s
volumes:
clickhouse_keeper_data:
clickhouse_01_data:
clickhouse_02_data:
kafka_data:
postgres_metadata_data:
airflow_logs:
airflow_auth:
superset_home:
prometheus_data:
grafana_data:
Reference in New Issue View Git Blame Copy Permalink