ddadmin
5270273d8a
- Why: - dashboard panels could resolve to stale datasource uid and show No data - monitoring required proactive alerts for ClickHouse health signals - What: - pin dashboard panels to prometheus_uid and remove datasource templating variable - fix PromQL metrics for CPU, inserted rows, and parts panels - add provisioning alert rules for failed queries, memory resident, and active parts - pin Prometheus datasource uid and update monitoring documentation - Check: - POST /api/admin/provisioning/datasources/reload - POST /api/admin/provisioning/dashboards/reload - POST /api/admin/provisioning/alerting/reload - GET /api/v1/provisioning/alert-rules
269 lines
7.7 KiB
YAML
269 lines
7.7 KiB
YAML
apiVersion: 1
|
|
|
|
groups:
|
|
- orgId: 1
|
|
name: clickhouse_health_group
|
|
folder: ClickHouse Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: ch_failed_queries_rate
|
|
title: ClickHouse Failed Queries Rate
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: prometheus_uid
|
|
model:
|
|
datasource:
|
|
type: prometheus
|
|
uid: prometheus_uid
|
|
editorMode: code
|
|
expr: rate(ClickHouseProfileEvents_FailedQuery[5m])
|
|
instant: false
|
|
intervalMs: 1000
|
|
legendFormat: __auto
|
|
maxDataPoints: 43200
|
|
range: true
|
|
refId: A
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator:
|
|
params: []
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
query:
|
|
params:
|
|
- B
|
|
reducer:
|
|
params: []
|
|
type: last
|
|
type: query
|
|
datasource:
|
|
type: __expr__
|
|
uid: __expr__
|
|
expression: A
|
|
intervalMs: 1000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
refId: B
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator:
|
|
params:
|
|
- 0
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
query:
|
|
params:
|
|
- C
|
|
reducer:
|
|
params: []
|
|
type: last
|
|
type: query
|
|
datasource:
|
|
type: __expr__
|
|
uid: __expr__
|
|
expression: B
|
|
intervalMs: 1000
|
|
maxDataPoints: 43200
|
|
refId: C
|
|
type: threshold
|
|
dashboardUid: clickhouse-overview
|
|
panelId: 8
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
for: 2m
|
|
annotations:
|
|
__dashboardUid__: clickhouse-overview
|
|
__panelId__: "8"
|
|
summary: "Есть ошибки запросов в ClickHouse"
|
|
runbook_url: "http://localhost:3000/d/clickhouse-overview/clickhouse-overview"
|
|
labels:
|
|
service: clickhouse
|
|
severity: warning
|
|
metric: failed_queries
|
|
isPaused: false
|
|
|
|
- uid: ch_memory_resident_high
|
|
title: ClickHouse Memory Resident High
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: prometheus_uid
|
|
model:
|
|
datasource:
|
|
type: prometheus
|
|
uid: prometheus_uid
|
|
editorMode: code
|
|
expr: 100 * ClickHouseAsyncMetrics_MemoryResident / ClickHouseAsyncMetrics_OSMemoryTotal
|
|
instant: false
|
|
intervalMs: 1000
|
|
legendFormat: __auto
|
|
maxDataPoints: 43200
|
|
range: true
|
|
refId: A
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator:
|
|
params: []
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
query:
|
|
params:
|
|
- B
|
|
reducer:
|
|
params: []
|
|
type: last
|
|
type: query
|
|
datasource:
|
|
type: __expr__
|
|
uid: __expr__
|
|
expression: A
|
|
intervalMs: 1000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
refId: B
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator:
|
|
params:
|
|
- 85
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
query:
|
|
params:
|
|
- C
|
|
reducer:
|
|
params: []
|
|
type: last
|
|
type: query
|
|
datasource:
|
|
type: __expr__
|
|
uid: __expr__
|
|
expression: B
|
|
intervalMs: 1000
|
|
maxDataPoints: 43200
|
|
refId: C
|
|
type: threshold
|
|
dashboardUid: clickhouse-overview
|
|
panelId: 3
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
for: 5m
|
|
annotations:
|
|
__dashboardUid__: clickhouse-overview
|
|
__panelId__: "3"
|
|
summary: "Высокое потребление памяти ClickHouse (Memory Resident)"
|
|
runbook_url: "http://localhost:3000/d/clickhouse-overview/clickhouse-overview"
|
|
labels:
|
|
service: clickhouse
|
|
severity: critical
|
|
metric: memory_resident
|
|
isPaused: false
|
|
|
|
- uid: ch_parts_active_high
|
|
title: ClickHouse Parts Active High
|
|
condition: C
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: prometheus_uid
|
|
model:
|
|
datasource:
|
|
type: prometheus
|
|
uid: prometheus_uid
|
|
editorMode: code
|
|
expr: ClickHouseMetrics_PartsActive
|
|
instant: false
|
|
intervalMs: 1000
|
|
legendFormat: __auto
|
|
maxDataPoints: 43200
|
|
range: true
|
|
refId: A
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator:
|
|
params: []
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
query:
|
|
params:
|
|
- B
|
|
reducer:
|
|
params: []
|
|
type: last
|
|
type: query
|
|
datasource:
|
|
type: __expr__
|
|
uid: __expr__
|
|
expression: A
|
|
intervalMs: 1000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
refId: B
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
conditions:
|
|
- evaluator:
|
|
params:
|
|
- 500
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
query:
|
|
params:
|
|
- C
|
|
reducer:
|
|
params: []
|
|
type: last
|
|
type: query
|
|
datasource:
|
|
type: __expr__
|
|
uid: __expr__
|
|
expression: B
|
|
intervalMs: 1000
|
|
maxDataPoints: 43200
|
|
refId: C
|
|
type: threshold
|
|
dashboardUid: clickhouse-overview
|
|
panelId: 12
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
for: 10m
|
|
annotations:
|
|
__dashboardUid__: clickhouse-overview
|
|
__panelId__: "12"
|
|
summary: "Слишком много активных MergeTree parts"
|
|
runbook_url: "http://localhost:3000/d/clickhouse-overview/clickhouse-overview"
|
|
labels:
|
|
service: clickhouse
|
|
severity: warning
|
|
metric: parts_active
|
|
isPaused: false
|