fix(monitoring): correct Kafka metrics and alert rules

- Why:
  - dashboard showed offset as throughput and produced misleading values
  - kafka-exporter metric/label naming was inconsistent across alerts/docs
  - consumer-group-missing alert was noisy for demo runs
- What:
  - switch throughput panel to rate(kafka_topic_partition_current_offset[5m]) aggregated by topic and exclude __* topics
  - align lag metric/labels to kafka_consumergroup_lag + consumergroup
  - remove Kafka Consumer Group Missing alert from provisioning
  - pin kafka-exporter image to v1.9.0 and update OPERATIONS.md checks
- Check:
  - airflow dags list-import-errors -> No data found
  - Prometheus targets: clickhouse up, kafka up
  - PromQL kafka_consumergroup_lag returns series
  - Grafana dashboards provisioning reload returns success
This commit is contained in:
2026-02-08 21:52:21 +03:00
parent afdfc98eef
commit 8e31b06241
4 changed files with 11 additions and 42 deletions
@@ -277,7 +277,7 @@
"type": "prometheus",
"uid": "prometheus_uid"
},
"expr": "count(count by (group) (kafka_consumergroup_lag))",
"expr": "count(count by (consumergroup) (kafka_consumergroup_lag))",
"refId": "A"
}
],
@@ -378,8 +378,8 @@
"type": "prometheus",
"uid": "prometheus_uid"
},
"expr": "kafka_topic_partition_current_offset",
"legendFormat": "{{topic}} - p{{partition}}",
"expr": "sum by(topic) (rate(kafka_topic_partition_current_offset{topic!~\"__.*\"}[5m]))",
"legendFormat": "{{topic}}",
"refId": "A"
}
],
@@ -478,7 +478,7 @@
"uid": "prometheus_uid"
},
"expr": "kafka_consumergroup_lag",
"legendFormat": "{{group}} - {{topic}}:{{partition}}",
"legendFormat": "{{consumergroup}} - {{topic}}:{{partition}}",
"refId": "A"
}
],