diff --git a/configs/grafana/provisioning/alerting/kafka-alert-rules.yml b/configs/grafana/provisioning/alerting/kafka-alert-rules.yml index d55fe43..4546f35 100644 --- a/configs/grafana/provisioning/alerting/kafka-alert-rules.yml +++ b/configs/grafana/provisioning/alerting/kafka-alert-rules.yml @@ -102,10 +102,10 @@ groups: type: prometheus uid: prometheus_uid editorMode: code - expr: kafka_consumer_group_lag + expr: kafka_consumergroup_lag instant: false intervalMs: 1000 - legendFormat: "{{ group }} - {{ topic }}:{{ partition }}" + legendFormat: "{{ consumergroup }} - {{ topic }}:{{ partition }}" maxDataPoints: 43200 range: true refId: A @@ -163,7 +163,7 @@ groups: execErrState: Error for: 5m annotations: - summary: "Высокий лаг консьюмера {{ $labels.group }}" + summary: "Высокий лаг консьюмера {{ $labels.consumergroup }}" description: "Lag для {{ $labels.topic }}:{{ $labels.partition }} составляет {{ $values.A }} сообщений (порог: 10000)" labels: severity: warning @@ -248,34 +248,3 @@ groups: description: "Rate сообщений < 0.1 msg/sec в течение 10 минут" labels: severity: warning - - # --- Alert: Consumer Group Missing --- - - uid: kafka_consumer_group_missing - title: Kafka Consumer Group Missing - condition: C - data: - - refId: A - relativeTimeRange: - from: 300 - to: 0 - datasourceUid: prometheus_uid - model: - datasource: - type: prometheus - uid: prometheus_uid - editorMode: code - expr: count by (group) (kafka_consumer_group_lag) - instant: true - intervalMs: 1000 - legendFormat: "{{ group }}" - maxDataPoints: 43200 - range: false - refId: A - noDataState: Alerting - execErrState: Error - for: 5m - annotations: - summary: "Пропала консьюмер-группа" - description: "Консьюмер-группа не обнаружена в течение 5 минут" - labels: - severity: info diff --git a/configs/grafana/provisioning/dashboards/kafka-overview.json b/configs/grafana/provisioning/dashboards/kafka-overview.json index 553a315..f9ffede 100644 --- a/configs/grafana/provisioning/dashboards/kafka-overview.json +++ b/configs/grafana/provisioning/dashboards/kafka-overview.json @@ -277,7 +277,7 @@ "type": "prometheus", "uid": "prometheus_uid" }, - "expr": "count(count by (group) (kafka_consumergroup_lag))", + "expr": "count(count by (consumergroup) (kafka_consumergroup_lag))", "refId": "A" } ], @@ -378,8 +378,8 @@ "type": "prometheus", "uid": "prometheus_uid" }, - "expr": "kafka_topic_partition_current_offset", - "legendFormat": "{{topic}} - p{{partition}}", + "expr": "sum by(topic) (rate(kafka_topic_partition_current_offset{topic!~\"__.*\"}[5m]))", + "legendFormat": "{{topic}}", "refId": "A" } ], @@ -478,7 +478,7 @@ "uid": "prometheus_uid" }, "expr": "kafka_consumergroup_lag", - "legendFormat": "{{group}} - {{topic}}:{{partition}}", + "legendFormat": "{{consumergroup}} - {{topic}}:{{partition}}", "refId": "A" } ], diff --git a/docker-compose.yml b/docker-compose.yml index 64402a2..23485df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -232,7 +232,7 @@ services: # Kafka Exporter для мониторинга через Prometheus kafka-exporter: - image: danielqsj/kafka-exporter:latest + image: danielqsj/kafka-exporter:v1.9.0 command: ["--kafka.server=kafka:29092"] ports: - "9308:9308" diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 5a4d768..1e344d9 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -200,7 +200,7 @@ curl -s "http://localhost:9090/api/v1/query?query=ClickHouseProfileEvents_Query" # Prometheus собирает метрики Kafka curl -s "http://localhost:9090/api/v1/query?query=kafka_brokers" -curl -s "http://localhost:9090/api/v1/query?query=kafka_consumer_group_lag" +curl -s "http://localhost:9090/api/v1/query?query=kafka_consumergroup_lag" # Прямая проверка kafka-exporter curl -s http://localhost:9308/metrics | grep "^kafka_" @@ -227,9 +227,9 @@ curl -s "http://localhost:9090/api/v1/query?query=ClickHouseProfileEvents_Query" Настроены правила: - `Kafka Broker Down` — `kafka_brokers < 1` в течение `1m` -- `Kafka Consumer Lag High` — `kafka_consumer_group_lag > 10000` в течение `5m` +- `Kafka Consumer Lag High` — `kafka_consumergroup_lag > 10000` в течение `5m` - `Kafka No Messages Produced` — `rate(kafka_topic_partition_current_offset[5m]) < 0.1` в течение `10m` -- `Kafka Consumer Group Missing` — пропала консьюмер-группа в течение `5m` +- `Kafka Consumer Group Missing` не включён: для демо-стенда даёт шум на стартовых прогонах и не повышает диагностику по сравнению с lag/throughput. Проверка и reload без рестарта контейнера: