fix(monitoring): correct Kafka metrics and alert rules

- Why:
  - dashboard showed offset as throughput and produced misleading values
  - kafka-exporter metric/label naming was inconsistent across alerts/docs
  - consumer-group-missing alert was noisy for demo runs
- What:
  - switch throughput panel to rate(kafka_topic_partition_current_offset[5m]) aggregated by topic and exclude __* topics
  - align lag metric/labels to kafka_consumergroup_lag + consumergroup
  - remove Kafka Consumer Group Missing alert from provisioning
  - pin kafka-exporter image to v1.9.0 and update OPERATIONS.md checks
- Check:
  - airflow dags list-import-errors -> No data found
  - Prometheus targets: clickhouse up, kafka up
  - PromQL kafka_consumergroup_lag returns series
  - Grafana dashboards provisioning reload returns success
This commit is contained in:
2026-02-08 21:52:21 +03:00
parent afdfc98eef
commit 8e31b06241
4 changed files with 11 additions and 42 deletions
@@ -102,10 +102,10 @@ groups:
type: prometheus
uid: prometheus_uid
editorMode: code
expr: kafka_consumer_group_lag
expr: kafka_consumergroup_lag
instant: false
intervalMs: 1000
legendFormat: "{{ group }} - {{ topic }}:{{ partition }}"
legendFormat: "{{ consumergroup }} - {{ topic }}:{{ partition }}"
maxDataPoints: 43200
range: true
refId: A
@@ -163,7 +163,7 @@ groups:
execErrState: Error
for: 5m
annotations:
summary: "Высокий лаг консьюмера {{ $labels.group }}"
summary: "Высокий лаг консьюмера {{ $labels.consumergroup }}"
description: "Lag для {{ $labels.topic }}:{{ $labels.partition }} составляет {{ $values.A }} сообщений (порог: 10000)"
labels:
severity: warning
@@ -248,34 +248,3 @@ groups:
description: "Rate сообщений < 0.1 msg/sec в течение 10 минут"
labels:
severity: warning
# --- Alert: Consumer Group Missing ---
- uid: kafka_consumer_group_missing
title: Kafka Consumer Group Missing
condition: C
data:
- refId: A
relativeTimeRange:
from: 300
to: 0
datasourceUid: prometheus_uid
model:
datasource:
type: prometheus
uid: prometheus_uid
editorMode: code
expr: count by (group) (kafka_consumer_group_lag)
instant: true
intervalMs: 1000
legendFormat: "{{ group }}"
maxDataPoints: 43200
range: false
refId: A
noDataState: Alerting
execErrState: Error
for: 5m
annotations:
summary: "Пропала консьюмер-группа"
description: "Консьюмер-группа не обнаружена в течение 5 минут"
labels:
severity: info