diff --git a/Makefile b/Makefile index edd4608..635e6e3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: up ddl data transform +.PHONY: up ddl data transform reload-monitoring COMPOSE ?= docker compose @@ -13,3 +13,16 @@ data: transform: bash ./scripts/run_batch.sh + +# Перезагрузка конфигурации мониторинга (после изменений в provisioning) +reload-monitoring: + @echo "=== Перезагрузка сервисов мониторинга ===" + $(COMPOSE) up -d prometheus grafana kafka-exporter + $(COMPOSE) restart prometheus + @echo "=== Перезагрузка provisioning Grafana ===" + @sleep 2 + @curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload && echo " [datasources]" + @curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload && echo " [dashboards]" + @curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload && echo " [alerting]" + @echo "=== Проверка ===" + @curl -s http://localhost:9090/api/v1/targets | grep -o '"job":"[^"]*"' | sort | uniq diff --git a/configs/grafana/provisioning/alerting/kafka-alert-rules.yml b/configs/grafana/provisioning/alerting/kafka-alert-rules.yml new file mode 100644 index 0000000..d55fe43 --- /dev/null +++ b/configs/grafana/provisioning/alerting/kafka-alert-rules.yml @@ -0,0 +1,281 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: kafka_health_group + folder: Kafka Alerts + interval: 1m + rules: + # --- Alert: Kafka Broker Down --- + - uid: kafka_broker_down + title: Kafka Broker Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus_uid + model: + datasource: + type: prometheus + uid: prometheus_uid + editorMode: code + expr: kafka_brokers + instant: false + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: Alerting + execErrState: Error + for: 1m + annotations: + summary: "Kafka broker недоступен" + description: "Количество доступных брокеров: {{ $values.A }} (ожидается ≥ 1)" + labels: + severity: critical + + # --- Alert: High Consumer Lag --- + - uid: kafka_consumer_lag_high + title: Kafka Consumer Lag High + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus_uid + model: + datasource: + type: prometheus + uid: prometheus_uid + editorMode: code + expr: kafka_consumer_group_lag + instant: false + intervalMs: 1000 + legendFormat: "{{ group }} - {{ topic }}:{{ partition }}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 10000 + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: "Высокий лаг консьюмера {{ $labels.group }}" + description: "Lag для {{ $labels.topic }}:{{ $labels.partition }} составляет {{ $values.A }} сообщений (порог: 10000)" + labels: + severity: warning + + # --- Alert: No Messages Produced --- + - uid: kafka_no_messages_produced + title: Kafka No Messages Produced + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus_uid + model: + datasource: + type: prometheus + uid: prometheus_uid + editorMode: code + expr: rate(kafka_topic_partition_current_offset[5m]) + instant: false + intervalMs: 1000 + legendFormat: "{{ topic }}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.1 + type: lt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 10m + annotations: + summary: "В топик {{ $labels.topic }} не поступают сообщения" + description: "Rate сообщений < 0.1 msg/sec в течение 10 минут" + labels: + severity: warning + + # --- Alert: Consumer Group Missing --- + - uid: kafka_consumer_group_missing + title: Kafka Consumer Group Missing + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus_uid + model: + datasource: + type: prometheus + uid: prometheus_uid + editorMode: code + expr: count by (group) (kafka_consumer_group_lag) + instant: true + intervalMs: 1000 + legendFormat: "{{ group }}" + maxDataPoints: 43200 + range: false + refId: A + noDataState: Alerting + execErrState: Error + for: 5m + annotations: + summary: "Пропала консьюмер-группа" + description: "Консьюмер-группа не обнаружена в течение 5 минут" + labels: + severity: info diff --git a/configs/grafana/provisioning/dashboards/kafka-overview.json b/configs/grafana/provisioning/dashboards/kafka-overview.json new file mode 100644 index 0000000..553a315 --- /dev/null +++ b/configs/grafana/provisioning/dashboards/kafka-overview.json @@ -0,0 +1,608 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Обзор метрик Kafka через kafka-exporter", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Cluster Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "kafka_brokers", + "refId": "A" + } + ], + "title": "Brokers Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "count(count by (topic) (kafka_topic_partitions))", + "refId": "A" + } + ], + "title": "Topics", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum(kafka_topic_partitions)", + "refId": "A" + } + ], + "title": "Total Partitions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "count(count by (group) (kafka_consumergroup_lag))", + "refId": "A" + } + ], + "title": "Consumer Groups", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 12, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "kafka_topic_partition_current_offset", + "legendFormat": "{{topic}} - p{{partition}}", + "refId": "A" + } + ], + "title": "Messages In / sec by Topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "red", + "value": 10000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "kafka_consumergroup_lag", + "legendFormat": "{{group}} - {{topic}}:{{partition}}", + "refId": "A" + } + ], + "title": "Consumer Lag by Group", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 13, + "panels": [], + "title": "Partitions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "kafka_topic_partition_current_offset", + "legendFormat": "{{topic}}:{{partition}}", + "refId": "A" + } + ], + "title": "Partition Offsets (Current)", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": ["kafka", "messaging", "de"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "Europe/Moscow", + "title": "Kafka Overview", + "uid": "kafka-overview", + "version": 1, + "weekStart": "" +} diff --git a/configs/prometheus.yml b/configs/prometheus.yml index 0b482fa..2e7fb3d 100644 --- a/configs/prometheus.yml +++ b/configs/prometheus.yml @@ -1,6 +1,6 @@ global: scrape_interval: 15s - + scrape_configs: - job_name: "clickhouse" metrics_path: "/metrics" @@ -8,4 +8,14 @@ scrape_configs: - targets: ["clickhouse:9126"] labels: instance: Clickhouse-1 + honor_labels: true + + # Kafka metrics via kafka-exporter + # Проверено через Context7: формат scrape_configs подтверждён (/prometheus/docs) + - job_name: "kafka" + metrics_path: "/metrics" + static_configs: + - targets: ["kafka-exporter:9308"] + labels: + instance: Kafka-1 honor_labels: true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d4804a2..9a7af85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -231,6 +231,18 @@ services: timeout: 10s retries: 5 + # Kafka Exporter для мониторинга через Prometheus + kafka-exporter: + image: danielqsj/kafka-exporter:latest + command: ["--kafka.server=kafka:29092"] + ports: + - "9308:9308" + networks: + - cs_dwh + depends_on: + - kafka + restart: unless-stopped + networks: cs_dwh: driver: bridge diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 61ac4af..188804d 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -93,7 +93,12 @@ docker compose exec -T clickhouse clickhouse-client --user=default --password=12 ### TL;DR после `git pull` ```bash -docker compose up -d prometheus grafana +# Быстрый вариант (make) +make reload-monitoring + +# Или вручную: +docker compose up -d prometheus grafana kafka-exporter +docker compose restart prometheus curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload @@ -127,17 +132,26 @@ curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"' Если прилетели изменения в `configs/grafana/provisioning/*` или `configs/prometheus.yml`, примените их так: ```bash -# Поднять/обновить сервисы мониторинга -docker compose up -d prometheus grafana +# Рекомендуемый способ (через make) +make reload-monitoring -# Перечитать provisioning Grafana без рестарта контейнера +# Или вручную: +docker compose up -d prometheus grafana kafka-exporter +docker compose restart prometheus curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload +``` -# Быстрая проверка, что ресурсы применились -curl -s -u admin:admin http://localhost:3000/api/datasources/name/Prometheus -curl -s -u admin:admin http://localhost:3000/api/v1/provisioning/alert-rules +Проверка результата: + +```bash +# Дашборды и алерты +curl -s -u admin:admin http://localhost:3000/api/v1/provisioning/alert-rules | grep -o '"title":"[^"]*"' + +# Kafka метрики +curl -s http://localhost:9090/api/v1/targets | grep kafka +curl -s http://localhost:9308/metrics | grep "^kafka_brokers" ``` Если в пулле изменился `configs/prometheus_ch.xml`, дополнительно перезапустите ClickHouse: @@ -146,6 +160,12 @@ curl -s -u admin:admin http://localhost:3000/api/v1/provisioning/alert-rules docker compose restart clickhouse ``` +Если дашборд Kafka не загрузился (ошибка "Dashboard title cannot be empty" в логах), пересоздайте контейнер Grafana: + +```bash +docker compose stop grafana && docker compose rm -f grafana && docker compose up -d grafana +``` + ### Дашборд ClickHouse Overview URL: `http://localhost:3000/d/clickhouse-overview/clickhouse-overview` @@ -158,8 +178,34 @@ URL: `http://localhost:3000/d/clickhouse-overview/clickhouse-overview` Принятое решение по метрикам: сверили naming через Context7 (`/clickhouse/clickhouse-docs`, раздел Prometheus interface) и заменили недоступные в `25.1` серии на фактически экспортируемые (`ClickHouseProfileEvents_InsertedRows`, `ClickHouseAsyncMetrics_TotalPartsOfMergeTreeTables`, `ClickHouseMetrics_Parts*`). +### Дашборд Kafka Overview + +URL: `http://localhost:3000/d/kafka-overview/kafka-overview` + +| Раздел | Метрики | +|--------|---------| +| Cluster Health | Brokers Up, Topics Count, Total Partitions, Consumer Groups | +| Throughput | Messages In/sec by Topic | +| Consumers | Consumer Lag by Group, Consumer Lag Table | +| Partitions | Partition Offsets (Current), Oldest vs Current Offset Gap | + +**Источник метрик:** `kafka-exporter` (danielqsj/kafka-exporter), формат конфигурации подтверждён через Context7 (`/danielqsj/kafka_exporter`, `/prometheus/docs`). + ### Проверка метрик +```bash +# Prometheus собирает метрики ClickHouse +curl -s "http://localhost:9090/api/v1/query?query=ClickHouseAsyncMetrics_MemoryResident" +curl -s "http://localhost:9090/api/v1/query?query=ClickHouseProfileEvents_Query" + +# Prometheus собирает метрики Kafka +curl -s "http://localhost:9090/api/v1/query?query=kafka_brokers" +curl -s "http://localhost:9090/api/v1/query?query=kafka_consumer_group_lag" + +# Прямая проверка kafka-exporter +curl -s http://localhost:9308/metrics | grep "^kafka_" +``` + ```bash # Проверить, что Prometheus собирает метрики curl -s "http://localhost:9090/api/v1/query?query=ClickHouseAsyncMetrics_MemoryResident" @@ -170,13 +216,21 @@ curl -s "http://localhost:9090/api/v1/query?query=ClickHouseProfileEvents_Query" ### Алерты Grafana -Provisioning-файл: `configs/grafana/provisioning/alerting/clickhouse-alert-rules.yml` +**ClickHouse Alerts** — provisioning-файл: `configs/grafana/provisioning/alerting/clickhouse-alert-rules.yml` Настроены правила: - `ClickHouse Failed Queries Rate` — `rate(ClickHouseProfileEvents_FailedQuery[5m]) > 0` в течение `2m` - `ClickHouse Memory Resident High` — `MemoryResident / OSMemoryTotal * 100 > 85` в течение `5m` - `ClickHouse Parts Active High` — `ClickHouseMetrics_PartsActive > 500` в течение `10m` +**Kafka Alerts** — provisioning-файл: `configs/grafana/provisioning/alerting/kafka-alert-rules.yml` + +Настроены правила: +- `Kafka Broker Down` — `kafka_brokers < 1` в течение `1m` +- `Kafka Consumer Lag High` — `kafka_consumer_group_lag > 10000` в течение `5m` +- `Kafka No Messages Produced` — `rate(kafka_topic_partition_current_offset[5m]) < 0.1` в течение `10m` +- `Kafka Consumer Group Missing` — пропала консьюмер-группа в течение `5m` + Проверка и reload без рестарта контейнера: ```bash @@ -189,10 +243,18 @@ curl -s -X POST -u admin:admin http://localhost:3000/api/admin/provisioning/aler ### Troubleshooting мониторинга +**Общие проблемы:** - **"No data" в Grafana**: проверить, что Prometheus видит target (`Status -> Targets` в UI) -- **Метрики не обновляются**: ClickHouse экспортирует метрики на `0.0.0.0:9126` внутри сети Docker - **Dashboard не загрузился**: проверить логи Grafana — provisioning работает при первом старте контейнера +**ClickHouse:** +- **Метрики не обновляются**: ClickHouse экспортирует метрики на `0.0.0.0:9126` внутри сети Docker + +**Kafka:** +- **`connection refused` к Kafka**: проверить, что kafka-exporter использует `kafka:29092` (внутренняя сеть), не `localhost:9092` +- **Метрики Kafka не появляются**: проверить, что kafka-exporter подключился к Kafka — `docker compose logs kafka-exporter` +- **Нет консьюмер-групп**: kafka-exporter показывает lag только при наличии активных консьюмеров с закоммиченными offset + ## Troubleshooting - `etl_pipeline` падает с ошибкой схемы: сначала запустить `ddl_init`. diff --git a/plans/monitoring_kafka_plan.md b/plans/monitoring_kafka_plan.md new file mode 100644 index 0000000..a67b82b --- /dev/null +++ b/plans/monitoring_kafka_plan.md @@ -0,0 +1,222 @@ +# План подключения Kafka к мониторингу + +> Дата создания: 2026-02-08 +> Источник: Kafka via danielqsj/kafka-exporter +> Проверка API: Context7 (`/danielqsj/kafka_exporter`, `/prometheus/docs`) + +--- + +## 1. Обзор + +Подключаем мониторинг Kafka через **kafka-exporter** (Prometheus exporter для Kafka). Используем официальный образ `danielqsj/kafka-exporter:latest`. + +**Метрики, которые будем собирать:** +- `kafka_brokers` — количество брокеров в кластере +- `kafka_topic_partitions` — количество партиций по топикам +- `kafka_topic_partition_current_offset` — текущий offset +- `kafka_topic_partition_oldest_offset` — oldest offset +- `kafka_consumer_group_lag` — лаг консьюмер-групп +- `kafka_consumer_group_current_offset` — текущий offset консьюмеров + +--- + +## 2. Изменения в инфраструктуре + +### 2.1. docker-compose.yml + +Добавить сервис `kafka-exporter`: + +```yaml + kafka-exporter: + image: danielqsj/kafka-exporter:latest + command: ["--kafka.server=kafka:29092"] + ports: + - "9308:9308" + networks: + - cs_dwh + depends_on: + - kafka +``` + +**Почему `kafka:29092`:** внутри Docker-сети Kafka слушает на `PLAINTEXT://kafka:29092` (см. `KAFKA_ADVERTISED_LISTENERS` в текущем docker-compose.yml). + +### 2.2. configs/prometheus.yml + +Добавить job для скрейпа kafka-exporter: + +```yaml +scrape_configs: + - job_name: "clickhouse" + metrics_path: "/metrics" + static_configs: + - targets: ["clickhouse:9126"] + labels: + instance: Clickhouse-1 + honor_labels: true + + # Новый job для Kafka + - job_name: "kafka" + metrics_path: "/metrics" + static_configs: + - targets: ["kafka-exporter:9308"] + labels: + instance: Kafka-1 + honor_labels: true +``` + +**Проверка через Context7:** формат `scrape_configs` подтверждён через `/prometheus/docs` (snippet: `scrape_configs: - job_name: myapp ... static_configs: - targets:`). + +--- + +## 3. Дашборд Grafana + +### 3.1. Создать файл: `configs/grafana/provisioning/dashboards/kafka-overview.json` + +Основные панели дашборда: + +| Панель | PromQL запрос | Описание | +|--------|--------------|----------| +| Brokers Up | `kafka_brokers` | Количество доступных брокеров | +| Topics Count | `count(kafka_topic_partitions) by (topic)` | Количество топиков | +| Total Partitions | `sum(kafka_topic_partitions)` | Всего партиций | +| Messages In/sec | `rate(kafka_topic_partition_current_offset[5m])` | Скорость записи сообщений | +| Consumer Lag | `kafka_consumer_group_lag` | Лаг по консьюмер-группам | +| Partition Offsets | `kafka_topic_partition_current_offset` | Текущие offsets по партициям | + +### 3.2. Структура дашборда (JSON) + +```json +{ + "dashboard": { + "id": null, + "uid": "kafka-overview", + "title": "Kafka Overview", + "tags": ["kafka", "messaging"], + "timezone": "Europe/Moscow", + "schemaVersion": 36, + "refresh": "10s", + "panels": [ + // Row 1: Cluster Health + // - Brokers Up (stat panel) + // - Topics Count (stat panel) + // - Total Partitions (stat panel) + + // Row 2: Throughput + // - Messages In/sec (graph panel) + // - Partition Offsets (graph panel) + + // Row 3: Consumers + // - Consumer Lag (table panel) + // - Consumer Offsets (graph panel) + ] + } +} +``` + +--- + +## 4. Алерт-правила Grafana + +### 4.1. Создать/дополнить: `configs/grafana/provisioning/alerting/kafka-alert-rules.yml` + +```yaml +apiVersion: 1 + +groups: + - orgId: 1 + name: kafka_health_group + folder: Kafka Alerts + interval: 1m + rules: + # Alert: Kafka broker down + - uid: kafka_broker_down + title: Kafka Broker Down + condition: C + data: + - refId: A + datasourceUid: prometheus_uid + model: + expr: kafka_brokers < 1 + noDataState: Alerting + for: 1m + annotations: + summary: "Kafka broker недоступен" + + # Alert: High consumer lag + - uid: kafka_consumer_lag_high + title: Kafka Consumer Lag High + condition: C + data: + - refId: A + datasourceUid: prometheus_uid + model: + expr: kafka_consumer_group_lag > 10000 + for: 5m + annotations: + summary: "Высокий лаг консьюмера {{ $labels.group }} для топика {{ $labels.topic }}" +``` + +--- + +## 5. Порядок внедрения + +```bash +# 1. Добавить сервис в docker-compose.yml +# 2. Обновить configs/prometheus.yml +# 3. Создать дашборд configs/grafana/provisioning/dashboards/kafka-overview.json +# 4. Создать алерты configs/grafana/provisioning/alerting/kafka-alert-rules.yml +# 5. Применить изменения + +docker compose up -d kafka-exporter +docker compose restart prometheus + +# Перезагрузить provisioning Grafana +curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload +curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload + +# Проверка +curl -s http://localhost:9090/api/v1/targets | grep kafka +curl -s http://localhost:9308/metrics | head -20 +``` + +--- + +## 6. URL после внедрения + +| Сервис | URL | +|--------|-----| +| Kafka Exporter metrics | http://localhost:9308/metrics | +| Grafana Dashboard | http://localhost:3000/d/kafka-overview/kafka-overview | +| Prometheus Targets | http://localhost:9090/targets | + +--- + +## 7. Проверка метрик + +```bash +# Проверить, что kafka-exporter отдаёт метрики +curl -s http://localhost:9308/metrics | grep "^kafka_" + +# Проверить, что Prometheus собирает метрики +curl -s "http://localhost:9090/api/v1/query?query=kafka_brokers" +``` + +--- + +## 8. Troubleshooting + +| Проблема | Решение | +|----------|---------| +| `connection refused` к Kafka | Проверить, что используется `kafka:29092` (внутренняя сеть), не `localhost:9092` | +| `No data` в Grafana | Проверить Status -> Targets в Prometheus UI | +| Алерты не загружаются | Проверить синтаксис YAML, перезагрузить provisioning | + +--- + +## 9. Принятые решения (фиксация) + +- **Образ kafka-exporter:** `danielqsj/kafka-exporter:latest` — стандарт de-facto, подтверждён через Context7. +- **Порт exporter:** `9308` (стандартный). +- **Адрес Kafka:** `kafka:29092` (внутренняя Docker-сеть, PLAINTEXT listener). +- **Формат конфигурации Prometheus:** подтверждён через Context7 (`/prometheus/docs`). +- **Без SASL/TLS:** для локального демо-стенда аутентификация не требуется.