ddmitry/clickstream-ch-kafka-superset-demo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitoring): add Grafana alert rules and fix datasource binding

Browse Source 
- Why:
  - dashboard panels could resolve to stale datasource uid and show No data
  - monitoring required proactive alerts for ClickHouse health signals
- What:
  - pin dashboard panels to prometheus_uid and remove datasource templating variable
  - fix PromQL metrics for CPU, inserted rows, and parts panels
  - add provisioning alert rules for failed queries, memory resident, and active parts
  - pin Prometheus datasource uid and update monitoring documentation
- Check:
  - POST /api/admin/provisioning/datasources/reload
  - POST /api/admin/provisioning/dashboards/reload
  - POST /api/admin/provisioning/alerting/reload
  - GET /api/v1/provisioning/alert-rules
This commit is contained in:
ddadmin
2026-02-08 20:49:31 +03:00
parent 140b711233
commit 5270273d8a
6 changed files with 326 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/grafana/provisioning/alerting/clickhouse-alert-rules.yml
+268
View File
@@ -0,0 +1,268 @@
apiVersion: 1
groups:
- orgId: 1
name: clickhouse_health_group
folder: ClickHouse Alerts
interval: 1m
rules:
- uid: ch_failed_queries_rate
title: ClickHouse Failed Queries Rate
condition: C
data:
- refId: A
relativeTimeRange:
from: 300
to: 0
datasourceUid: prometheus_uid
model:
datasource:
type: prometheus
uid: prometheus_uid
editorMode: code
expr: rate(ClickHouseProfileEvents_FailedQuery[5m])
instant: false
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: true
refId: A
- refId: B
datasourceUid: __expr__
model:
conditions:
- evaluator:
params: []
type: gt
operator:
type: and
query:
params:
- B
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
reducer: last
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
type: gt
operator:
type: and
query:
params:
- C
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
dashboardUid: clickhouse-overview
panelId: 8
noDataState: NoData
execErrState: Error
for: 2m
annotations:
__dashboardUid__: clickhouse-overview
__panelId__: "8"
summary: "Есть ошибки запросов в ClickHouse"
runbook_url: "http://localhost:3000/d/clickhouse-overview/clickhouse-overview"
labels:
service: clickhouse
severity: warning
metric: failed_queries
isPaused: false
- uid: ch_memory_resident_high
title: ClickHouse Memory Resident High
condition: C
data:
- refId: A
relativeTimeRange:
from: 300
to: 0
datasourceUid: prometheus_uid
model:
datasource:
type: prometheus
uid: prometheus_uid
editorMode: code
expr: 100 * ClickHouseAsyncMetrics_MemoryResident / ClickHouseAsyncMetrics_OSMemoryTotal
instant: false
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: true
refId: A
- refId: B
datasourceUid: __expr__
model:
conditions:
- evaluator:
params: []
type: gt
operator:
type: and
query:
params:
- B
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
reducer: last
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 85
type: gt
operator:
type: and
query:
params:
- C
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
dashboardUid: clickhouse-overview
panelId: 3
noDataState: NoData
execErrState: Error
for: 5m
annotations:
__dashboardUid__: clickhouse-overview
__panelId__: "3"
summary: "Высокое потребление памяти ClickHouse (Memory Resident)"
runbook_url: "http://localhost:3000/d/clickhouse-overview/clickhouse-overview"
labels:
service: clickhouse
severity: critical
metric: memory_resident
isPaused: false
- uid: ch_parts_active_high
title: ClickHouse Parts Active High
condition: C
data:
- refId: A
relativeTimeRange:
from: 300
to: 0
datasourceUid: prometheus_uid
model:
datasource:
type: prometheus
uid: prometheus_uid
editorMode: code
expr: ClickHouseMetrics_PartsActive
instant: false
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: true
refId: A
- refId: B
datasourceUid: __expr__
model:
conditions:
- evaluator:
params: []
type: gt
operator:
type: and
query:
params:
- B
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
reducer: last
refId: B
type: reduce
- refId: C
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 500
type: gt
operator:
type: and
query:
params:
- C
reducer:
params: []
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: B
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
dashboardUid: clickhouse-overview
panelId: 12
noDataState: NoData
execErrState: Error
for: 10m
annotations:
__dashboardUid__: clickhouse-overview
__panelId__: "12"
summary: "Слишком много активных MergeTree parts"
runbook_url: "http://localhost:3000/d/clickhouse-overview/clickhouse-overview"
labels:
service: clickhouse
severity: warning
metric: parts_active
isPaused: false
configs/grafana/provisioning/dashboards/clickhouse-overview.json
+27 -40
View File
@@ -38,7 +38,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -116,7 +116,7 @@
},
"targets": [
{
"expr": "rate(ClickHouseAsyncMetrics_CPUStealMicroseconds[1m]) / 10000",
"expr": "rate(ClickHouseProfileEvents_OSCPUVirtualTimeMicroseconds[1m]) / 1000000 / scalar(count({__name__=~\"ClickHouseAsyncMetrics_CPUFrequencyMHz_.*\"})) * 100",
"refId": "A"
}
],
@@ -126,7 +126,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -210,7 +210,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -307,7 +307,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -391,7 +391,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -475,7 +475,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -536,7 +536,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -593,7 +593,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -667,7 +667,7 @@
},
"targets": [
{
"expr": "rate(ClickHouseProfileEvents_InsertedRowsCount[1m])",
"expr": "rate(ClickHouseProfileEvents_InsertedRows[1m])",
"refId": "A"
}
],
@@ -690,7 +690,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -737,7 +737,7 @@
"pluginVersion": "11.5.2",
"targets": [
{
"expr": "ClickHouseMetrics_Parts",
"expr": "ClickHouseAsyncMetrics_TotalPartsOfMergeTreeTables",
"refId": "A"
}
],
@@ -747,7 +747,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -803,6 +803,7 @@
"gridPos": {
"h": 8,
"w": 16,
"x": 8,
"y": 23
},
"id": 13,
@@ -820,18 +821,18 @@
},
"targets": [
{
"expr": "ClickHouseMetrics_Parts",
"legendFormat": "{{ database }}.{{ table }}",
"expr": "{__name__=~\"ClickHouseMetrics_Parts(Active|Committed|Outdated|Deleting|PreActive|PreCommitted|Temporary|Wide|Compact|DeleteOnDestroy)\"}",
"legendFormat": "{{ __name__ }}",
"refId": "A"
}
],
"title": "Parts by Table",
"title": "Parts by State",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -888,7 +889,7 @@
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
"uid": "prometheus_uid"
},
"fieldConfig": {
"defaults": {
@@ -972,27 +973,13 @@
],
"refresh": "30s",
"schemaVersion": 39,
"tags": ["clickhouse", "dwh", "monitoring"],
"tags": [
"clickhouse",
"dwh",
"monitoring"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
"list": []
},
"time": {
"from": "now-1h",
@@ -1002,6 +989,6 @@
"timezone": "Europe/Moscow",
"title": "ClickHouse Overview",
"uid": "clickhouse-overview",
"version": 2,
"version": 3,
"weekStart": ""
}
}
configs/grafana/provisioning/datasources/prometheus.yml
+1
View File
@@ -7,6 +7,7 @@ apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus_uid
type: prometheus
access: proxy
url: http://prometheus:9090