clickstream-data-platform/infra/grafana/provisioning/dashboards/data.json

{
  "uid": "data",
  "title": "Данные",
  "timezone": "utc",
  "schemaVersion": 42,
  "refresh": "1m",
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "panels": [
    {
      "type": "stat",
      "title": "Свежесть: сколько прошло с последней доехавшей строки",
      "datasource": {
        "type": "grafana-clickhouse-datasource",
        "uid": "clickhouse"
      },
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 0,
        "y": 0
      },
      "options": {
        "colorMode": "none",
        "graphMode": "none"
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "clickhouse"
          },
          "editorType": "sql",
          "queryType": "table",
          "format": 1,
          "rawSql": "SELECT\n    (SELECT dateDiff('second', max(_load_ts), now()) FROM ods.event_dist) AS \"события\",\n    (SELECT dateDiff('second', max(_load_ts), now()) FROM ods.order_dist) AS \"заказы\""
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      }
    },
    {
      "type": "table",
      "title": "Брак: сколько приехавшего не разобралось",
      "datasource": {
        "type": "grafana-clickhouse-datasource",
        "uid": "clickhouse"
      },
      "gridPos": {
        "h": 5,
        "w": 9,
        "x": 6,
        "y": 0
      },
      "options": {},
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "clickhouse"
          },
          "editorType": "sql",
          "queryType": "table",
          "format": 1,
          "rawSql": "SELECT\n    source AS \"Источник\",\n    countIf(NOT is_error) AS \"Приехало\",\n    countIf(is_error) AS \"Из них в брак\",\n    100 * countIf(is_error) / countIf(NOT is_error) AS \"Доля брака, %\"\nFROM\n(\n    SELECT 'события' AS source, 0 AS is_error, _load_ts FROM stg.hits_raw_dist\n    UNION ALL\n    SELECT 'события', 1, _load_ts FROM ods.event_errors_dist\n    UNION ALL\n    SELECT 'заказы', 0, _load_ts FROM stg.orders_raw_dist\n    UNION ALL\n    SELECT 'заказы', 1, _load_ts FROM ods.order_errors_dist\n)\nWHERE $__timeFilter(_load_ts)\nGROUP BY source\nORDER BY source"
        }
      ]
    },
    {
      "type": "table",
      "title": "Дневной слепок заказов: какой доехал, когда и после какой паузы",
      "datasource": {
        "type": "grafana-clickhouse-datasource",
        "uid": "clickhouse"
      },
      "gridPos": {
        "h": 5,
        "w": 9,
        "x": 15,
        "y": 0
      },
      "options": {},
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "clickhouse"
          },
          "editorType": "sql",
          "queryType": "table",
          "format": 1,
          "rawSql": "SELECT\n    \"Слепок\",\n    \"Доехал\",\n    dateDiff('second', prev, \"Доехал\") AS \"После прошлого, с\",\n    \"Строк\"\nFROM\n(\n    SELECT\n        JSONExtractString(raw, 'snapshot_date') AS \"Слепок\",\n        max(_load_ts) AS \"Доехал\",\n        count() AS \"Строк\",\n        -- Обнуляемый тип нужен, чтобы у самого раннего слепка пауза осталась\n        -- пустой, а не превратилась в разницу с началом эпохи.\n        lagInFrame(toNullable(max(_load_ts))) OVER (ORDER BY \"Слепок\") AS prev\n    FROM stg.orders_raw_dist\n    GROUP BY \"Слепок\"\n)\nORDER BY \"Слепок\" DESC\nLIMIT 10"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Поток: сколько строк приезжает в сырой слой",
      "datasource": {
        "type": "grafana-clickhouse-datasource",
        "uid": "clickhouse"
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 5
      },
      "options": {
        "tooltip": {
          "mode": "multi"
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "clickhouse"
          },
          "editorType": "sql",
          "queryType": "timeseries",
          "format": 0,
          "rawSql": "SELECT\n    $__timeInterval(_load_ts) AS t,\n    countIf(source = 'события') AS \"события\",\n    countIf(source = 'заказы') AS \"заказы\"\nFROM\n(\n    SELECT 'события' AS source, _load_ts FROM stg.hits_raw_dist\n    UNION ALL\n    SELECT 'заказы', _load_ts FROM stg.orders_raw_dist\n)\nWHERE $__timeFilter(_load_ts)\nGROUP BY t\nORDER BY t"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Сходимость: произведено в Kafka против приземлившегося в ClickHouse",
      "datasource": {
        "type": "datasource",
        "uid": "-- Mixed --"
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 5
      },
      "options": {
        "tooltip": {
          "mode": "multi"
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(kafka_topic_partition_current_offset{topic=\"hits\"}) - sum(kafka_topic_partition_current_offset{topic=\"hits\"} offset $__interval)",
          "legendFormat": "произведено в Kafka",
          "range": true
        },
        {
          "refId": "B",
          "datasource": {
            "type": "grafana-clickhouse-datasource",
            "uid": "clickhouse"
          },
          "editorType": "sql",
          "queryType": "timeseries",
          "format": 0,
          "rawSql": "SELECT\n    -- Правый край, а не левый: ряд Kafka в этой точке — прирост за тот же\n    -- интервал, а он набирается к его концу.\n    $__timeInterval(_load_ts) + INTERVAL $__interval_s SECOND AS t,\n    count() AS \"приземлилось в ClickHouse\"\nFROM stg.hits_raw_dist\nWHERE $__timeFilter(_load_ts)\nGROUP BY t\nORDER BY t"
        }
      ],
      "fieldConfig": {
        "defaults": {},
        "overrides": [
          {
            "matcher": {
              "id": "byFrameRefID",
              "options": "B"
            },
            "properties": [
              {
                "id": "displayName",
                "value": "приземлилось в ClickHouse"
              }
            ]
          }
        ]
      },
      "interval": "1m"
    },
    {
      "type": "timeseries",
      "title": "Отставание чтения из Kafka: непрочитанных сообщений",
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "gridPos": {
        "h": 7,
        "w": 24,
        "x": 0,
        "y": 14
      },
      "options": {
        "tooltip": {
          "mode": "multi"
        }
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum by (consumergroup) (kafka_consumergroup_lag)",
          "legendFormat": "{{consumergroup}}",
          "range": true
        }
      ]
    }
  ]
}