feat(infra): стенд целиком — Kafka, Airflow 3, Superset и мониторинг одной командой
Зачем: рядом с кластером ClickHouse не хватало остальной платформы, а поднимать её по кускам — значит каждый раз вспоминать порядок. Теперь `make up` даёт стенд целиком, а `make smoke` честно отвечает, работает он или нет. Что: - Kafka в режиме KRaft (без ZooKeeper), Postgres под метаданные, Airflow 3.3 четырьмя сервисами и Superset 6.1 с драйверами ClickHouse; - мониторинг: Prometheus снимает метрики с обеих нод ClickHouse и keeper, Grafana получает подготовленный источник данных; - подключение Airflow ведёт на ноду 1, подключение Superset — на ноду 2: ловушка правильных ошибок, забытый `ON CLUSTER` виден в дашборде сам; - `scripts/stand-smoke.sh` — сквозная проверка из 24 пунктов: топик в Kafka, цели Prometheus, запуск примера DAG через API Airflow, проверка подключения Superset и расход памяти против порога 3,4 ГБ; - `make config-test` — статические ворота: Compose, синтаксис Bash и Python, стражи README; стражи smoke проверяют, что отчёт краснеет на сломанном стенде и зеленеет после восстановления; - решения записаны в `docs/adr/0001-stand-services.md`, состав стенда и порядок работы — в README. Проверка: на чистых томах `make clean` → `cp .env.example .env` → `make up` (1 мин 51 с) → `make smoke` — 24 пройдено, 0 ошибок, 2171 MiB. Перезапуск `make down` → `make up` → `make smoke` — 24/0. Также зелены `make config-test` (3/0 и 5/0), `make smoke-cluster` (8/8) и `make smoke-guards` (3/0 и 3/0). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+258
@@ -23,6 +23,48 @@ x-clickhouse-common: &clickhouse-common
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
|
||||
x-airflow-common: &airflow-common
|
||||
image: ${AIRFLOW_IMAGE:-apache/airflow:3.3.0}
|
||||
environment:
|
||||
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
||||
AIRFLOW__CORE__PARALLELISM: "4"
|
||||
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
|
||||
AIRFLOW__CORE__AUTH_MANAGER: airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager
|
||||
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: "${AIRFLOW_ADMIN_USER:-admin}:admin"
|
||||
AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/auth/simple_auth_manager_passwords.json
|
||||
AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-apiserver:8080/execution/
|
||||
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=}
|
||||
AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_JWT_SECRET:-local-only-airflow-jwt-secret-do-not-use}
|
||||
AIRFLOW__API_AUTH__JWT_ISSUER: airflow
|
||||
AIRFLOW__API__SECRET_KEY: ${AIRFLOW_API_SECRET_KEY:-local-only-airflow-api-secret-do-not-use}
|
||||
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: "postgresql+psycopg2://${AIRFLOW_METADATA_USER:-airflow}:${AIRFLOW_METADATA_PASSWORD:-airflow-local}@postgres-metadata:5432/airflow"
|
||||
AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true"
|
||||
AIRFLOW_ADMIN_USER: ${AIRFLOW_ADMIN_USER:-admin}
|
||||
AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-airflow}
|
||||
volumes:
|
||||
- ./dags:/opt/airflow/dags:ro
|
||||
- ./infra/airflow/init.sh:/opt/airflow/init.sh:ro
|
||||
- airflow_logs:/opt/airflow/logs
|
||||
- airflow_auth:/opt/airflow/auth
|
||||
|
||||
x-superset-common: &superset-common
|
||||
image: clickstream-superset:local
|
||||
build:
|
||||
context: ./infra/superset
|
||||
args:
|
||||
SUPERSET_BASE_IMAGE: ${SUPERSET_IMAGE:-apache/superset:6.1.0}
|
||||
environment:
|
||||
SUPERSET_CONFIG_PATH: /app/pythonpath/superset_config.py
|
||||
SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY:-local-only-superset-secret-do-not-use-outside-this-stand}
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://${SUPERSET_METADATA_USER:-superset}:${SUPERSET_METADATA_PASSWORD:-superset-local}@postgres-metadata:5432/superset"
|
||||
SUPERSET_ADMIN_USER: ${SUPERSET_ADMIN_USER:-admin}
|
||||
SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD:-superset}
|
||||
volumes:
|
||||
- ./infra/superset/superset_config.py:/app/pythonpath/superset_config.py:ro
|
||||
- ./infra/superset/init.sh:/app/infra/init.sh:ro
|
||||
- ./infra/superset/import:/app/import:ro
|
||||
- superset_home:/app/superset_home
|
||||
|
||||
services:
|
||||
clickhouse-keeper:
|
||||
image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.3.17.56}
|
||||
@@ -52,6 +94,7 @@ services:
|
||||
- "127.0.0.1:${CLICKHOUSE_01_TCP_PORT:-29000}:9000"
|
||||
volumes:
|
||||
- ./infra/clickhouse/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
|
||||
- ./infra/clickhouse/config.d/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
|
||||
- ./infra/clickhouse/config.d/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml:ro
|
||||
- clickhouse_01_data:/var/lib/clickhouse
|
||||
|
||||
@@ -63,10 +106,225 @@ services:
|
||||
- "127.0.0.1:${CLICKHOUSE_02_TCP_PORT:-29001}:9000"
|
||||
volumes:
|
||||
- ./infra/clickhouse/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
|
||||
- ./infra/clickhouse/config.d/prometheus.xml:/etc/clickhouse-server/config.d/prometheus.xml:ro
|
||||
- ./infra/clickhouse/config.d/macros-02.xml:/etc/clickhouse-server/config.d/macros.xml:ro
|
||||
- clickhouse_02_data:/var/lib/clickhouse
|
||||
|
||||
kafka:
|
||||
image: ${KAFKA_IMAGE:-apache/kafka:4.3.1}
|
||||
hostname: kafka
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${KAFKA_EXTERNAL_PORT:-29092}:29092"
|
||||
environment:
|
||||
KAFKA_NODE_ID: "1"
|
||||
KAFKA_PROCESS_ROLES: "broker,controller"
|
||||
KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
|
||||
KAFKA_LISTENERS: "INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:29092"
|
||||
KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:9092,EXTERNAL://localhost:${KAFKA_EXTERNAL_PORT:-29092}"
|
||||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT"
|
||||
KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
|
||||
KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
|
||||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
|
||||
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"
|
||||
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1"
|
||||
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: "1"
|
||||
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: "1"
|
||||
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: "0"
|
||||
KAFKA_NUM_PARTITIONS: "1"
|
||||
KAFKA_LOG_DIRS: "/var/lib/kafka/data"
|
||||
KAFKA_HEAP_OPTS: "-Xmx512m -Xms256m"
|
||||
CLUSTER_ID: "4L6g3nShT-eMCtK--X86sw"
|
||||
volumes:
|
||||
- kafka_data:/var/lib/kafka/data
|
||||
mem_limit: 1g
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list >/dev/null 2>&1"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
start_period: 30s
|
||||
|
||||
postgres-metadata:
|
||||
image: ${POSTGRES_IMAGE:-postgres:16-alpine}
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- shared_buffers=64MB
|
||||
- -c
|
||||
- max_connections=50
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_ADMIN_USER:-postgres}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD:-postgres-local}
|
||||
POSTGRES_DB: postgres
|
||||
AIRFLOW_METADATA_USER: ${AIRFLOW_METADATA_USER:-airflow}
|
||||
AIRFLOW_METADATA_PASSWORD: ${AIRFLOW_METADATA_PASSWORD:-airflow-local}
|
||||
SUPERSET_METADATA_USER: ${SUPERSET_METADATA_USER:-superset}
|
||||
SUPERSET_METADATA_PASSWORD: ${SUPERSET_METADATA_PASSWORD:-superset-local}
|
||||
volumes:
|
||||
- ./infra/postgres/init/10-metadata-databases.sh:/docker-entrypoint-initdb.d/10-metadata-databases.sh:ro
|
||||
- postgres_metadata_data:/var/lib/postgresql/data
|
||||
mem_limit: 256m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d postgres >/dev/null"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
|
||||
airflow-init:
|
||||
<<: *airflow-common
|
||||
user: "0:0"
|
||||
restart: "no"
|
||||
depends_on:
|
||||
postgres-metadata:
|
||||
condition: service_healthy
|
||||
clickhouse-01:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/bin/bash"]
|
||||
command: ["/opt/airflow/init.sh"]
|
||||
|
||||
airflow-apiserver:
|
||||
<<: *airflow-common
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
airflow-init:
|
||||
condition: service_completed_successfully
|
||||
command: api-server
|
||||
ports:
|
||||
- "127.0.0.1:${AIRFLOW_PORT:-28080}:8080"
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8080/api/v2/monitor/health | python -c 'import json,sys; sys.exit(0 if json.load(sys.stdin)[\"metadatabase\"][\"status\"] == \"healthy\" else 1)'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 30s
|
||||
|
||||
airflow-scheduler:
|
||||
<<: *airflow-common
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
airflow-init:
|
||||
condition: service_completed_successfully
|
||||
command: scheduler
|
||||
mem_limit: 640m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8974/health >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 30s
|
||||
|
||||
airflow-dag-processor:
|
||||
<<: *airflow-common
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
airflow-init:
|
||||
condition: service_completed_successfully
|
||||
airflow-apiserver:
|
||||
condition: service_healthy
|
||||
command: dag-processor
|
||||
mem_limit: 384m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://airflow-apiserver:8080/api/v2/monitor/health | python -c 'import json,sys; sys.exit(0 if json.load(sys.stdin)[\"dag_processor\"][\"status\"] == \"healthy\" else 1)'"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 30s
|
||||
|
||||
superset-init:
|
||||
<<: *superset-common
|
||||
restart: "no"
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
postgres-metadata:
|
||||
condition: service_healthy
|
||||
clickhouse-02:
|
||||
condition: service_healthy
|
||||
entrypoint: ["/bin/bash"]
|
||||
command: ["/app/infra/init.sh"]
|
||||
|
||||
superset:
|
||||
<<: *superset-common
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
superset-init:
|
||||
condition: service_completed_successfully
|
||||
ports:
|
||||
- "127.0.0.1:${SUPERSET_PORT:-28088}:8088"
|
||||
mem_limit: 1g
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8088/health >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 30s
|
||||
|
||||
prometheus:
|
||||
image: ${PROMETHEUS_IMAGE:-prom/prometheus:v3.13.2}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
clickhouse-keeper:
|
||||
condition: service_healthy
|
||||
clickhouse-01:
|
||||
condition: service_healthy
|
||||
clickhouse-02:
|
||||
condition: service_healthy
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --storage.tsdb.retention.size=512MB
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:${PROMETHEUS_PORT:-29090}:9090"
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9090/-/ready >/dev/null 2>&1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 15
|
||||
start_period: 10s
|
||||
|
||||
grafana:
|
||||
image: ${GRAFANA_IMAGE:-grafana/grafana:13.1.1}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-23000}:3000"
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:3000/api/health >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 15
|
||||
start_period: 15s
|
||||
|
||||
volumes:
|
||||
clickhouse_keeper_data:
|
||||
clickhouse_01_data:
|
||||
clickhouse_02_data:
|
||||
kafka_data:
|
||||
postgres_metadata_data:
|
||||
airflow_logs:
|
||||
airflow_auth:
|
||||
superset_home:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
|
||||
Reference in New Issue
Block a user