feat(monitoring): add Kafka monitoring via kafka-exporter
- Add kafka-exporter service to docker-compose.yml - Add kafka job to prometheus.yml scrape configs - Add Kafka Overview dashboard (Grafana provisioning) - Add Kafka alert rules (broker down, consumer lag, etc.) - Add make reload-monitoring command for easy updates - Update OPERATIONS.md with TL;DR and troubleshooting API verified via Context7: - /danielqsj/kafka_exporter for exporter config - /prometheus/docs for scrape_configs format
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
.PHONY: up ddl data transform
|
||||
.PHONY: up ddl data transform reload-monitoring
|
||||
|
||||
COMPOSE ?= docker compose
|
||||
|
||||
@@ -13,3 +13,16 @@ data:
|
||||
|
||||
transform:
|
||||
bash ./scripts/run_batch.sh
|
||||
|
||||
# Перезагрузка конфигурации мониторинга (после изменений в provisioning)
|
||||
reload-monitoring:
|
||||
@echo "=== Перезагрузка сервисов мониторинга ==="
|
||||
$(COMPOSE) up -d prometheus grafana kafka-exporter
|
||||
$(COMPOSE) restart prometheus
|
||||
@echo "=== Перезагрузка provisioning Grafana ==="
|
||||
@sleep 2
|
||||
@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload && echo " [datasources]"
|
||||
@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload && echo " [dashboards]"
|
||||
@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload && echo " [alerting]"
|
||||
@echo "=== Проверка ==="
|
||||
@curl -s http://localhost:9090/api/v1/targets | grep -o '"job":"[^"]*"' | sort | uniq
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: kafka_health_group
|
||||
folder: Kafka Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
# --- Alert: Kafka Broker Down ---
|
||||
- uid: kafka_broker_down
|
||||
title: Kafka Broker Down
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 300
|
||||
to: 0
|
||||
datasourceUid: prometheus_uid
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus_uid
|
||||
editorMode: code
|
||||
expr: kafka_brokers
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: __auto
|
||||
maxDataPoints: 43200
|
||||
range: true
|
||||
refId: A
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 1
|
||||
type: lt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- B
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: B
|
||||
type: threshold
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 0
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- C
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: B
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
noDataState: Alerting
|
||||
execErrState: Error
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Kafka broker недоступен"
|
||||
description: "Количество доступных брокеров: {{ $values.A }} (ожидается ≥ 1)"
|
||||
labels:
|
||||
severity: critical
|
||||
|
||||
# --- Alert: High Consumer Lag ---
|
||||
- uid: kafka_consumer_lag_high
|
||||
title: Kafka Consumer Lag High
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 300
|
||||
to: 0
|
||||
datasourceUid: prometheus_uid
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus_uid
|
||||
editorMode: code
|
||||
expr: kafka_consumer_group_lag
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{ group }} - {{ topic }}:{{ partition }}"
|
||||
maxDataPoints: 43200
|
||||
range: true
|
||||
refId: A
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 10000
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- B
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: B
|
||||
type: threshold
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 0
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- C
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: B
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Высокий лаг консьюмера {{ $labels.group }}"
|
||||
description: "Lag для {{ $labels.topic }}:{{ $labels.partition }} составляет {{ $values.A }} сообщений (порог: 10000)"
|
||||
labels:
|
||||
severity: warning
|
||||
|
||||
# --- Alert: No Messages Produced ---
|
||||
- uid: kafka_no_messages_produced
|
||||
title: Kafka No Messages Produced
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: prometheus_uid
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus_uid
|
||||
editorMode: code
|
||||
expr: rate(kafka_topic_partition_current_offset[5m])
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{ topic }}"
|
||||
maxDataPoints: 43200
|
||||
range: true
|
||||
refId: A
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 0.1
|
||||
type: lt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- B
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: B
|
||||
type: threshold
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- 0
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- C
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: B
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
annotations:
|
||||
summary: "В топик {{ $labels.topic }} не поступают сообщения"
|
||||
description: "Rate сообщений < 0.1 msg/sec в течение 10 минут"
|
||||
labels:
|
||||
severity: warning
|
||||
|
||||
# --- Alert: Consumer Group Missing ---
|
||||
- uid: kafka_consumer_group_missing
|
||||
title: Kafka Consumer Group Missing
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 300
|
||||
to: 0
|
||||
datasourceUid: prometheus_uid
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus_uid
|
||||
editorMode: code
|
||||
expr: count by (group) (kafka_consumer_group_lag)
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{ group }}"
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
noDataState: Alerting
|
||||
execErrState: Error
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Пропала консьюмер-группа"
|
||||
description: "Консьюмер-группа не обнаружена в течение 5 минут"
|
||||
labels:
|
||||
severity: info
|
||||
@@ -0,0 +1,608 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Обзор метрик Kafka через kafka-exporter",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 11,
|
||||
"panels": [],
|
||||
"title": "Cluster Health",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "11.0.8",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"expr": "kafka_brokers",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Brokers Up",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "continuous-GrYlRd"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "11.0.8",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"expr": "count(count by (topic) (kafka_topic_partitions))",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Topics",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "continuous-BlPu"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "11.0.8",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"expr": "sum(kafka_topic_partitions)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Total Partitions",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "11.0.8",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"expr": "count(count by (group) (kafka_consumergroup_lag))",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Consumer Groups",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"id": 12,
|
||||
"panels": [],
|
||||
"title": "Throughput",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "cps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"maxHeight": 600,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"expr": "kafka_topic_partition_current_offset",
|
||||
"legendFormat": "{{topic}} - p{{partition}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Messages In / sec by Topic",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 1000
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 10000
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 6
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"maxHeight": 600,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "11.0.8",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"expr": "kafka_consumergroup_lag",
|
||||
"legendFormat": "{{group}} - {{topic}}:{{partition}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Consumer Lag by Group",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"id": 13,
|
||||
"panels": [],
|
||||
"title": "Partitions",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"maxHeight": 600,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "11.0.8",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus_uid"
|
||||
},
|
||||
"expr": "kafka_topic_partition_current_offset",
|
||||
"legendFormat": "{{topic}}:{{partition}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Partition Offsets (Current)",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["kafka", "messaging", "de"],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "Europe/Moscow",
|
||||
"title": "Kafka Overview",
|
||||
"uid": "kafka-overview",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
+11
-1
@@ -1,6 +1,6 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "clickhouse"
|
||||
metrics_path: "/metrics"
|
||||
@@ -8,4 +8,14 @@ scrape_configs:
|
||||
- targets: ["clickhouse:9126"]
|
||||
labels:
|
||||
instance: Clickhouse-1
|
||||
honor_labels: true
|
||||
|
||||
# Kafka metrics via kafka-exporter
|
||||
# Проверено через Context7: формат scrape_configs подтверждён (/prometheus/docs)
|
||||
- job_name: "kafka"
|
||||
metrics_path: "/metrics"
|
||||
static_configs:
|
||||
- targets: ["kafka-exporter:9308"]
|
||||
labels:
|
||||
instance: Kafka-1
|
||||
honor_labels: true
|
||||
@@ -231,6 +231,18 @@ services:
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
# Kafka Exporter для мониторинга через Prometheus
|
||||
kafka-exporter:
|
||||
image: danielqsj/kafka-exporter:latest
|
||||
command: ["--kafka.server=kafka:29092"]
|
||||
ports:
|
||||
- "9308:9308"
|
||||
networks:
|
||||
- cs_dwh
|
||||
depends_on:
|
||||
- kafka
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
cs_dwh:
|
||||
driver: bridge
|
||||
|
||||
+71
-9
@@ -93,7 +93,12 @@ docker compose exec -T clickhouse clickhouse-client --user=default --password=12
|
||||
### TL;DR после `git pull`
|
||||
|
||||
```bash
|
||||
docker compose up -d prometheus grafana
|
||||
# Быстрый вариант (make)
|
||||
make reload-monitoring
|
||||
|
||||
# Или вручную:
|
||||
docker compose up -d prometheus grafana kafka-exporter
|
||||
docker compose restart prometheus
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload
|
||||
@@ -127,17 +132,26 @@ curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"'
|
||||
Если прилетели изменения в `configs/grafana/provisioning/*` или `configs/prometheus.yml`, примените их так:
|
||||
|
||||
```bash
|
||||
# Поднять/обновить сервисы мониторинга
|
||||
docker compose up -d prometheus grafana
|
||||
# Рекомендуемый способ (через make)
|
||||
make reload-monitoring
|
||||
|
||||
# Перечитать provisioning Grafana без рестарта контейнера
|
||||
# Или вручную:
|
||||
docker compose up -d prometheus grafana kafka-exporter
|
||||
docker compose restart prometheus
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload
|
||||
```
|
||||
|
||||
# Быстрая проверка, что ресурсы применились
|
||||
curl -s -u admin:admin http://localhost:3000/api/datasources/name/Prometheus
|
||||
curl -s -u admin:admin http://localhost:3000/api/v1/provisioning/alert-rules
|
||||
Проверка результата:
|
||||
|
||||
```bash
|
||||
# Дашборды и алерты
|
||||
curl -s -u admin:admin http://localhost:3000/api/v1/provisioning/alert-rules | grep -o '"title":"[^"]*"'
|
||||
|
||||
# Kafka метрики
|
||||
curl -s http://localhost:9090/api/v1/targets | grep kafka
|
||||
curl -s http://localhost:9308/metrics | grep "^kafka_brokers"
|
||||
```
|
||||
|
||||
Если в пулле изменился `configs/prometheus_ch.xml`, дополнительно перезапустите ClickHouse:
|
||||
@@ -146,6 +160,12 @@ curl -s -u admin:admin http://localhost:3000/api/v1/provisioning/alert-rules
|
||||
docker compose restart clickhouse
|
||||
```
|
||||
|
||||
Если дашборд Kafka не загрузился (ошибка "Dashboard title cannot be empty" в логах), пересоздайте контейнер Grafana:
|
||||
|
||||
```bash
|
||||
docker compose stop grafana && docker compose rm -f grafana && docker compose up -d grafana
|
||||
```
|
||||
|
||||
### Дашборд ClickHouse Overview
|
||||
|
||||
URL: `http://localhost:3000/d/clickhouse-overview/clickhouse-overview`
|
||||
@@ -158,8 +178,34 @@ URL: `http://localhost:3000/d/clickhouse-overview/clickhouse-overview`
|
||||
|
||||
Принятое решение по метрикам: сверили naming через Context7 (`/clickhouse/clickhouse-docs`, раздел Prometheus interface) и заменили недоступные в `25.1` серии на фактически экспортируемые (`ClickHouseProfileEvents_InsertedRows`, `ClickHouseAsyncMetrics_TotalPartsOfMergeTreeTables`, `ClickHouseMetrics_Parts*`).
|
||||
|
||||
### Дашборд Kafka Overview
|
||||
|
||||
URL: `http://localhost:3000/d/kafka-overview/kafka-overview`
|
||||
|
||||
| Раздел | Метрики |
|
||||
|--------|---------|
|
||||
| Cluster Health | Brokers Up, Topics Count, Total Partitions, Consumer Groups |
|
||||
| Throughput | Messages In/sec by Topic |
|
||||
| Consumers | Consumer Lag by Group, Consumer Lag Table |
|
||||
| Partitions | Partition Offsets (Current), Oldest vs Current Offset Gap |
|
||||
|
||||
**Источник метрик:** `kafka-exporter` (danielqsj/kafka-exporter), формат конфигурации подтверждён через Context7 (`/danielqsj/kafka_exporter`, `/prometheus/docs`).
|
||||
|
||||
### Проверка метрик
|
||||
|
||||
```bash
|
||||
# Prometheus собирает метрики ClickHouse
|
||||
curl -s "http://localhost:9090/api/v1/query?query=ClickHouseAsyncMetrics_MemoryResident"
|
||||
curl -s "http://localhost:9090/api/v1/query?query=ClickHouseProfileEvents_Query"
|
||||
|
||||
# Prometheus собирает метрики Kafka
|
||||
curl -s "http://localhost:9090/api/v1/query?query=kafka_brokers"
|
||||
curl -s "http://localhost:9090/api/v1/query?query=kafka_consumer_group_lag"
|
||||
|
||||
# Прямая проверка kafka-exporter
|
||||
curl -s http://localhost:9308/metrics | grep "^kafka_"
|
||||
```
|
||||
|
||||
```bash
|
||||
# Проверить, что Prometheus собирает метрики
|
||||
curl -s "http://localhost:9090/api/v1/query?query=ClickHouseAsyncMetrics_MemoryResident"
|
||||
@@ -170,13 +216,21 @@ curl -s "http://localhost:9090/api/v1/query?query=ClickHouseProfileEvents_Query"
|
||||
|
||||
### Алерты Grafana
|
||||
|
||||
Provisioning-файл: `configs/grafana/provisioning/alerting/clickhouse-alert-rules.yml`
|
||||
**ClickHouse Alerts** — provisioning-файл: `configs/grafana/provisioning/alerting/clickhouse-alert-rules.yml`
|
||||
|
||||
Настроены правила:
|
||||
- `ClickHouse Failed Queries Rate` — `rate(ClickHouseProfileEvents_FailedQuery[5m]) > 0` в течение `2m`
|
||||
- `ClickHouse Memory Resident High` — `MemoryResident / OSMemoryTotal * 100 > 85` в течение `5m`
|
||||
- `ClickHouse Parts Active High` — `ClickHouseMetrics_PartsActive > 500` в течение `10m`
|
||||
|
||||
**Kafka Alerts** — provisioning-файл: `configs/grafana/provisioning/alerting/kafka-alert-rules.yml`
|
||||
|
||||
Настроены правила:
|
||||
- `Kafka Broker Down` — `kafka_brokers < 1` в течение `1m`
|
||||
- `Kafka Consumer Lag High` — `kafka_consumer_group_lag > 10000` в течение `5m`
|
||||
- `Kafka No Messages Produced` — `rate(kafka_topic_partition_current_offset[5m]) < 0.1` в течение `10m`
|
||||
- `Kafka Consumer Group Missing` — пропала консьюмер-группа в течение `5m`
|
||||
|
||||
Проверка и reload без рестарта контейнера:
|
||||
|
||||
```bash
|
||||
@@ -189,10 +243,18 @@ curl -s -X POST -u admin:admin http://localhost:3000/api/admin/provisioning/aler
|
||||
|
||||
### Troubleshooting мониторинга
|
||||
|
||||
**Общие проблемы:**
|
||||
- **"No data" в Grafana**: проверить, что Prometheus видит target (`Status -> Targets` в UI)
|
||||
- **Метрики не обновляются**: ClickHouse экспортирует метрики на `0.0.0.0:9126` внутри сети Docker
|
||||
- **Dashboard не загрузился**: проверить логи Grafana — provisioning работает при первом старте контейнера
|
||||
|
||||
**ClickHouse:**
|
||||
- **Метрики не обновляются**: ClickHouse экспортирует метрики на `0.0.0.0:9126` внутри сети Docker
|
||||
|
||||
**Kafka:**
|
||||
- **`connection refused` к Kafka**: проверить, что kafka-exporter использует `kafka:29092` (внутренняя сеть), не `localhost:9092`
|
||||
- **Метрики Kafka не появляются**: проверить, что kafka-exporter подключился к Kafka — `docker compose logs kafka-exporter`
|
||||
- **Нет консьюмер-групп**: kafka-exporter показывает lag только при наличии активных консьюмеров с закоммиченными offset
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- `etl_pipeline` падает с ошибкой схемы: сначала запустить `ddl_init`.
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
# План подключения Kafka к мониторингу
|
||||
|
||||
> Дата создания: 2026-02-08
|
||||
> Источник: Kafka via danielqsj/kafka-exporter
|
||||
> Проверка API: Context7 (`/danielqsj/kafka_exporter`, `/prometheus/docs`)
|
||||
|
||||
---
|
||||
|
||||
## 1. Обзор
|
||||
|
||||
Подключаем мониторинг Kafka через **kafka-exporter** (Prometheus exporter для Kafka). Используем официальный образ `danielqsj/kafka-exporter:latest`.
|
||||
|
||||
**Метрики, которые будем собирать:**
|
||||
- `kafka_brokers` — количество брокеров в кластере
|
||||
- `kafka_topic_partitions` — количество партиций по топикам
|
||||
- `kafka_topic_partition_current_offset` — текущий offset
|
||||
- `kafka_topic_partition_oldest_offset` — oldest offset
|
||||
- `kafka_consumer_group_lag` — лаг консьюмер-групп
|
||||
- `kafka_consumer_group_current_offset` — текущий offset консьюмеров
|
||||
|
||||
---
|
||||
|
||||
## 2. Изменения в инфраструктуре
|
||||
|
||||
### 2.1. docker-compose.yml
|
||||
|
||||
Добавить сервис `kafka-exporter`:
|
||||
|
||||
```yaml
|
||||
kafka-exporter:
|
||||
image: danielqsj/kafka-exporter:latest
|
||||
command: ["--kafka.server=kafka:29092"]
|
||||
ports:
|
||||
- "9308:9308"
|
||||
networks:
|
||||
- cs_dwh
|
||||
depends_on:
|
||||
- kafka
|
||||
```
|
||||
|
||||
**Почему `kafka:29092`:** внутри Docker-сети Kafka слушает на `PLAINTEXT://kafka:29092` (см. `KAFKA_ADVERTISED_LISTENERS` в текущем docker-compose.yml).
|
||||
|
||||
### 2.2. configs/prometheus.yml
|
||||
|
||||
Добавить job для скрейпа kafka-exporter:
|
||||
|
||||
```yaml
|
||||
scrape_configs:
|
||||
- job_name: "clickhouse"
|
||||
metrics_path: "/metrics"
|
||||
static_configs:
|
||||
- targets: ["clickhouse:9126"]
|
||||
labels:
|
||||
instance: Clickhouse-1
|
||||
honor_labels: true
|
||||
|
||||
# Новый job для Kafka
|
||||
- job_name: "kafka"
|
||||
metrics_path: "/metrics"
|
||||
static_configs:
|
||||
- targets: ["kafka-exporter:9308"]
|
||||
labels:
|
||||
instance: Kafka-1
|
||||
honor_labels: true
|
||||
```
|
||||
|
||||
**Проверка через Context7:** формат `scrape_configs` подтверждён через `/prometheus/docs` (snippet: `scrape_configs: - job_name: myapp ... static_configs: - targets:`).
|
||||
|
||||
---
|
||||
|
||||
## 3. Дашборд Grafana
|
||||
|
||||
### 3.1. Создать файл: `configs/grafana/provisioning/dashboards/kafka-overview.json`
|
||||
|
||||
Основные панели дашборда:
|
||||
|
||||
| Панель | PromQL запрос | Описание |
|
||||
|--------|--------------|----------|
|
||||
| Brokers Up | `kafka_brokers` | Количество доступных брокеров |
|
||||
| Topics Count | `count(kafka_topic_partitions) by (topic)` | Количество топиков |
|
||||
| Total Partitions | `sum(kafka_topic_partitions)` | Всего партиций |
|
||||
| Messages In/sec | `rate(kafka_topic_partition_current_offset[5m])` | Скорость записи сообщений |
|
||||
| Consumer Lag | `kafka_consumer_group_lag` | Лаг по консьюмер-группам |
|
||||
| Partition Offsets | `kafka_topic_partition_current_offset` | Текущие offsets по партициям |
|
||||
|
||||
### 3.2. Структура дашборда (JSON)
|
||||
|
||||
```json
|
||||
{
|
||||
"dashboard": {
|
||||
"id": null,
|
||||
"uid": "kafka-overview",
|
||||
"title": "Kafka Overview",
|
||||
"tags": ["kafka", "messaging"],
|
||||
"timezone": "Europe/Moscow",
|
||||
"schemaVersion": 36,
|
||||
"refresh": "10s",
|
||||
"panels": [
|
||||
// Row 1: Cluster Health
|
||||
// - Brokers Up (stat panel)
|
||||
// - Topics Count (stat panel)
|
||||
// - Total Partitions (stat panel)
|
||||
|
||||
// Row 2: Throughput
|
||||
// - Messages In/sec (graph panel)
|
||||
// - Partition Offsets (graph panel)
|
||||
|
||||
// Row 3: Consumers
|
||||
// - Consumer Lag (table panel)
|
||||
// - Consumer Offsets (graph panel)
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Алерт-правила Grafana
|
||||
|
||||
### 4.1. Создать/дополнить: `configs/grafana/provisioning/alerting/kafka-alert-rules.yml`
|
||||
|
||||
```yaml
|
||||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: kafka_health_group
|
||||
folder: Kafka Alerts
|
||||
interval: 1m
|
||||
rules:
|
||||
# Alert: Kafka broker down
|
||||
- uid: kafka_broker_down
|
||||
title: Kafka Broker Down
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
datasourceUid: prometheus_uid
|
||||
model:
|
||||
expr: kafka_brokers < 1
|
||||
noDataState: Alerting
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Kafka broker недоступен"
|
||||
|
||||
# Alert: High consumer lag
|
||||
- uid: kafka_consumer_lag_high
|
||||
title: Kafka Consumer Lag High
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
datasourceUid: prometheus_uid
|
||||
model:
|
||||
expr: kafka_consumer_group_lag > 10000
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Высокий лаг консьюмера {{ $labels.group }} для топика {{ $labels.topic }}"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Порядок внедрения
|
||||
|
||||
```bash
|
||||
# 1. Добавить сервис в docker-compose.yml
|
||||
# 2. Обновить configs/prometheus.yml
|
||||
# 3. Создать дашборд configs/grafana/provisioning/dashboards/kafka-overview.json
|
||||
# 4. Создать алерты configs/grafana/provisioning/alerting/kafka-alert-rules.yml
|
||||
# 5. Применить изменения
|
||||
|
||||
docker compose up -d kafka-exporter
|
||||
docker compose restart prometheus
|
||||
|
||||
# Перезагрузить provisioning Grafana
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload
|
||||
curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload
|
||||
|
||||
# Проверка
|
||||
curl -s http://localhost:9090/api/v1/targets | grep kafka
|
||||
curl -s http://localhost:9308/metrics | head -20
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. URL после внедрения
|
||||
|
||||
| Сервис | URL |
|
||||
|--------|-----|
|
||||
| Kafka Exporter metrics | http://localhost:9308/metrics |
|
||||
| Grafana Dashboard | http://localhost:3000/d/kafka-overview/kafka-overview |
|
||||
| Prometheus Targets | http://localhost:9090/targets |
|
||||
|
||||
---
|
||||
|
||||
## 7. Проверка метрик
|
||||
|
||||
```bash
|
||||
# Проверить, что kafka-exporter отдаёт метрики
|
||||
curl -s http://localhost:9308/metrics | grep "^kafka_"
|
||||
|
||||
# Проверить, что Prometheus собирает метрики
|
||||
curl -s "http://localhost:9090/api/v1/query?query=kafka_brokers"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Troubleshooting
|
||||
|
||||
| Проблема | Решение |
|
||||
|----------|---------|
|
||||
| `connection refused` к Kafka | Проверить, что используется `kafka:29092` (внутренняя сеть), не `localhost:9092` |
|
||||
| `No data` в Grafana | Проверить Status -> Targets в Prometheus UI |
|
||||
| Алерты не загружаются | Проверить синтаксис YAML, перезагрузить provisioning |
|
||||
|
||||
---
|
||||
|
||||
## 9. Принятые решения (фиксация)
|
||||
|
||||
- **Образ kafka-exporter:** `danielqsj/kafka-exporter:latest` — стандарт de-facto, подтверждён через Context7.
|
||||
- **Порт exporter:** `9308` (стандартный).
|
||||
- **Адрес Kafka:** `kafka:29092` (внутренняя Docker-сеть, PLAINTEXT listener).
|
||||
- **Формат конфигурации Prometheus:** подтверждён через Context7 (`/prometheus/docs`).
|
||||
- **Без SASL/TLS:** для локального демо-стенда аутентификация не требуется.
|
||||
Reference in New Issue
Block a user