ddadmin
d15f4fe4d8
Зачем: до запуска менти должен понимать действие параметров, а после ошибки проверки — где искать причину. Что: уточнены тексты Airflow, самопроверки, DQ и run-dag.sh, подписи Superset и Grafana. Логика, запросы, метрики и идентификаторы сохранены. Проверка: make lint, git diff --check, bash -n; разбор YAML/JSON, сравнение структуры Python и конфигурации, искусственные примеры ошибок. Airflow и Grafana проверены через локальные API, Superset — по YAML. Визуальная проверка интерфейсов не выполнена: браузер недоступен.
320 lines
10 KiB
JSON
320 lines
10 KiB
JSON
{
|
|
"uid": "cluster",
|
|
"title": "Кластер",
|
|
"timezone": "utc",
|
|
"schemaVersion": 42,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-30m",
|
|
"to": "now"
|
|
},
|
|
"panels": [
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Процессор: сколько ядер использует узел",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"options": {
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"editorType": "sql",
|
|
"queryType": "timeseries",
|
|
"format": 0,
|
|
"rawSql": "SELECT\n $__timeInterval(event_time) AS t,\n hostName() AS node,\n -- Процессорное время самого сервера. Метрики с именами OS* сняты с машины\n -- целиком, и в них обе ноды показывали бы одно и то же число.\n sum(ProfileEvent_OSCPUVirtualTimeMicroseconds) / 1e6 / $__interval_s AS \"ядер\"\nFROM clusterAllReplicas('clickstream_cluster', system.metric_log)\nWHERE $__timeFilter(event_time)\nGROUP BY t, node\nORDER BY t, node"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"displayName": "${__field.labels.node}",
|
|
"min": 0,
|
|
"decimals": 2
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Память: использование и лимит узла",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 0
|
|
},
|
|
"options": {
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"editorType": "sql",
|
|
"queryType": "timeseries",
|
|
"format": 0,
|
|
"rawSql": "SELECT\n $__timeInterval(event_time) AS t,\n hostName() AS node,\n -- Занятое меряется резидентной памятью: именно её ядро сравнивает с\n -- пределом контейнера, когда решает, убить ли ноду.\n maxIf(value, metric = 'MemoryResident') AS \"занято\",\n maxIf(value, metric = 'CGroupMemoryTotal') AS \"отведено\"\nFROM clusterAllReplicas('clickstream_cluster', system.asynchronous_metric_log)\nWHERE $__timeFilter(event_time)\n AND metric IN ('MemoryResident', 'CGroupMemoryTotal')\nGROUP BY t, node\nORDER BY t, node"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"displayName": "${__field.name} · ${__field.labels.node}",
|
|
"min": 0,
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byRegexp",
|
|
"options": "отведено.*"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.lineStyle",
|
|
"value": {
|
|
"fill": "dash",
|
|
"dash": [
|
|
10,
|
|
10
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "custom.fillOpacity",
|
|
"value": 0
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Диски: скорость чтения и записи узла",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 0
|
|
},
|
|
"options": {
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"editorType": "sql",
|
|
"queryType": "timeseries",
|
|
"format": 0,
|
|
"rawSql": "SELECT\n $__timeInterval(event_time) AS t,\n hostName() AS node,\n -- Байты, прошедшие через обращения к файлам: чтение из страничного кэша\n -- тоже видно. Счётчики блочного устройства на вставке молчат.\n sum(ProfileEvent_OSReadChars) / $__interval_s AS \"чтение\",\n sum(ProfileEvent_OSWriteChars) / $__interval_s AS \"запись\"\nFROM clusterAllReplicas('clickstream_cluster', system.metric_log)\nWHERE $__timeFilter(event_time)\nGROUP BY t, node\nORDER BY t, node"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"displayName": "${__field.name} · ${__field.labels.node}",
|
|
"min": 0,
|
|
"unit": "Bps"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Части таблиц: количество и среднее число строк в части",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"options": {},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"editorType": "sql",
|
|
"queryType": "table",
|
|
"format": 1,
|
|
"rawSql": "SELECT\n concat(database, '.', table) AS \"Таблица\",\n hostName() AS \"Нода\",\n count() AS \"Частей\",\n sum(rows) AS \"Строк\",\n round(sum(rows) / count()) AS \"Строк в части\",\n sum(bytes_on_disk) AS \"На диске\"\nFROM clusterAllReplicas('clickstream_cluster', system.parts)\nWHERE active\n AND database != 'system'\nGROUP BY \"Таблица\", \"Нода\"\nORDER BY \"Таблица\", \"Нода\"\n"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "На диске"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "unit",
|
|
"value": "bytes"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Слияния: число объединенных частей",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"options": {
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"editorType": "sql",
|
|
"queryType": "timeseries",
|
|
"format": 0,
|
|
"rawSql": "SELECT\n $__timeInterval(event_time) AS t,\n database AS db,\n sum(length(merged_from)) AS \"частей съедено\"\nFROM clusterAllReplicas('clickstream_cluster', system.part_log)\nWHERE $__timeFilter(event_time)\n AND event_type = 'MergeParts'\n AND database != 'system'\nGROUP BY t, db\nORDER BY t, db"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"displayName": "${__field.labels.db}",
|
|
"min": 0,
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 60,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"interval": "1m"
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Keeper: сколько узлов в дереве",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"options": {
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "ClickHouseAsyncMetrics_KeeperZnodeCount",
|
|
"legendFormat": "узлов в дереве",
|
|
"range": true
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"min": 0
|
|
},
|
|
"overrides": []
|
|
},
|
|
"description": "Узлы дерева Keeper (znode) — записи со служебными данными координации ClickHouse."
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Распределенный DDL: результаты команд на узлах кластера",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 16,
|
|
"x": 8,
|
|
"y": 16
|
|
},
|
|
"options": {},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "grafana-clickhouse-datasource",
|
|
"uid": "clickhouse"
|
|
},
|
|
"editorType": "sql",
|
|
"queryType": "table",
|
|
"format": 1,
|
|
"rawSql": "SELECT\n query_create_time AS \"Создан\",\n entry AS \"Запись\",\n host AS \"Нода\",\n status AS \"Состояние\",\n query_duration_ms AS \"Заняло, мс\",\n exception_text AS \"Ошибка\",\n query AS \"Запрос\"\nFROM system.distributed_ddl_queue\nORDER BY entry DESC, \"Нода\"\nLIMIT 20"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
}
|
|
]
|
|
}
|