ddadmin
d15f4fe4d8
Зачем: до запуска менти должен понимать действие параметров, а после ошибки проверки — где искать причину. Что: уточнены тексты Airflow, самопроверки, DQ и run-dag.sh, подписи Superset и Grafana. Логика, запросы, метрики и идентификаторы сохранены. Проверка: make lint, git diff --check, bash -n; разбор YAML/JSON, сравнение структуры Python и конфигурации, искусственные примеры ошибок. Airflow и Grafana проверены через локальные API, Superset — по YAML. Визуальная проверка интерфейсов не выполнена: браузер недоступен.
266 lines
8.3 KiB
JSON
266 lines
8.3 KiB
JSON
{
|
||
"uid": "data",
|
||
"title": "Данные",
|
||
"timezone": "utc",
|
||
"schemaVersion": 42,
|
||
"refresh": "1m",
|
||
"time": {
|
||
"from": "now-1h",
|
||
"to": "now"
|
||
},
|
||
"panels": [
|
||
{
|
||
"type": "stat",
|
||
"title": "Свежесть: время с последней загрузки в ODS",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"gridPos": {
|
||
"h": 5,
|
||
"w": 6,
|
||
"x": 0,
|
||
"y": 0
|
||
},
|
||
"options": {
|
||
"colorMode": "none",
|
||
"graphMode": "none"
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"editorType": "sql",
|
||
"queryType": "table",
|
||
"format": 1,
|
||
"rawSql": "SELECT\n (SELECT dateDiff('second', max(_load_ts), now()) FROM ods.event_dist) AS \"события\",\n (SELECT dateDiff('second', max(_load_ts), now()) FROM ods.order_dist) AS \"заказы\""
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "s"
|
||
},
|
||
"overrides": []
|
||
}
|
||
},
|
||
{
|
||
"type": "table",
|
||
"title": "Брак: доля строк с ошибками разбора",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"gridPos": {
|
||
"h": 5,
|
||
"w": 9,
|
||
"x": 6,
|
||
"y": 0
|
||
},
|
||
"options": {},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"editorType": "sql",
|
||
"queryType": "table",
|
||
"format": 1,
|
||
"rawSql": "SELECT\n source AS \"Источник\",\n countIf(NOT is_error) AS \"Приехало\",\n countIf(is_error) AS \"Из них в брак\",\n 100 * countIf(is_error) / countIf(NOT is_error) AS \"Доля брака, %\"\nFROM\n(\n SELECT 'события' AS source, 0 AS is_error, _load_ts FROM stg.hits_raw_dist\n UNION ALL\n SELECT 'события', 1, _load_ts FROM ods.event_errors_dist\n UNION ALL\n SELECT 'заказы', 0, _load_ts FROM stg.orders_raw_dist\n UNION ALL\n SELECT 'заказы', 1, _load_ts FROM ods.order_errors_dist\n)\nWHERE $__timeFilter(_load_ts)\nGROUP BY source\nORDER BY source"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Приехало"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "displayName",
|
||
"value": "Загружено в STG"
|
||
}
|
||
]
|
||
}
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"type": "table",
|
||
"title": "Дневной слепок заказов: дата и время загрузки",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"gridPos": {
|
||
"h": 5,
|
||
"w": 9,
|
||
"x": 15,
|
||
"y": 0
|
||
},
|
||
"options": {},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"editorType": "sql",
|
||
"queryType": "table",
|
||
"format": 1,
|
||
"rawSql": "SELECT\n \"Слепок\",\n \"Доехал\",\n dateDiff('second', prev, \"Доехал\") AS \"После прошлого, с\",\n \"Строк\"\nFROM\n(\n SELECT\n JSONExtractString(raw, 'snapshot_date') AS \"Слепок\",\n max(_load_ts) AS \"Доехал\",\n count() AS \"Строк\",\n -- Обнуляемый тип нужен, чтобы у самого раннего слепка пауза осталась\n -- пустой, а не превратилась в разницу с началом эпохи.\n lagInFrame(toNullable(max(_load_ts))) OVER (ORDER BY \"Слепок\") AS prev\n FROM stg.orders_raw_dist\n GROUP BY \"Слепок\"\n)\nORDER BY \"Слепок\" DESC\nLIMIT 10"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Доехал"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "displayName",
|
||
"value": "Время загрузки"
|
||
}
|
||
]
|
||
}
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"type": "timeseries",
|
||
"title": "Поток: число сообщений, загруженных в STG",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"gridPos": {
|
||
"h": 9,
|
||
"w": 12,
|
||
"x": 0,
|
||
"y": 5
|
||
},
|
||
"options": {
|
||
"tooltip": {
|
||
"mode": "multi"
|
||
}
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"editorType": "sql",
|
||
"queryType": "timeseries",
|
||
"format": 0,
|
||
"rawSql": "SELECT\n $__timeInterval(_load_ts) AS t,\n countIf(source = 'события') AS \"события\",\n countIf(source = 'заказы') AS \"заказы\"\nFROM\n(\n SELECT 'события' AS source, _load_ts FROM stg.hits_raw_dist\n UNION ALL\n SELECT 'заказы', _load_ts FROM stg.orders_raw_dist\n)\nWHERE $__timeFilter(_load_ts)\nGROUP BY t\nORDER BY t"
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"type": "timeseries",
|
||
"title": "Сходимость: число событий в Kafka и ClickHouse",
|
||
"datasource": {
|
||
"type": "datasource",
|
||
"uid": "-- Mixed --"
|
||
},
|
||
"gridPos": {
|
||
"h": 9,
|
||
"w": 12,
|
||
"x": 12,
|
||
"y": 5
|
||
},
|
||
"options": {
|
||
"tooltip": {
|
||
"mode": "multi"
|
||
}
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "prometheus"
|
||
},
|
||
"editorMode": "code",
|
||
"expr": "sum(kafka_topic_partition_current_offset{topic=\"hits\"}) - sum(kafka_topic_partition_current_offset{topic=\"hits\"} offset $__interval)",
|
||
"legendFormat": "поступило в Kafka",
|
||
"range": true
|
||
},
|
||
{
|
||
"refId": "B",
|
||
"datasource": {
|
||
"type": "grafana-clickhouse-datasource",
|
||
"uid": "clickhouse"
|
||
},
|
||
"editorType": "sql",
|
||
"queryType": "timeseries",
|
||
"format": 0,
|
||
"rawSql": "SELECT\n -- Правый край, а не левый: ряд Kafka в этой точке — прирост за тот же\n -- интервал, а он набирается к его концу.\n $__timeInterval(_load_ts) + INTERVAL $__interval_s SECOND AS t,\n count() AS \"приземлилось в ClickHouse\"\nFROM stg.hits_raw_dist\nWHERE $__timeFilter(_load_ts)\nGROUP BY t\nORDER BY t"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byFrameRefID",
|
||
"options": "B"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "displayName",
|
||
"value": "загружено в ClickHouse"
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"interval": "1m"
|
||
},
|
||
{
|
||
"type": "timeseries",
|
||
"title": "Отставание чтения из Kafka: число непрочитанных сообщений",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "prometheus"
|
||
},
|
||
"gridPos": {
|
||
"h": 7,
|
||
"w": 24,
|
||
"x": 0,
|
||
"y": 14
|
||
},
|
||
"options": {
|
||
"tooltip": {
|
||
"mode": "multi"
|
||
}
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "prometheus"
|
||
},
|
||
"editorMode": "code",
|
||
"expr": "sum by (consumergroup) (kafka_consumergroup_lag)",
|
||
"legendFormat": "{{consumergroup}}",
|
||
"range": true
|
||
}
|
||
]
|
||
}
|
||
]
|
||
}
|