From d97be5fa5621c21d131b92b2cbe3b6c8a7b387fe Mon Sep 17 00:00:00 2001 From: Dmitry Dementev Date: Sun, 15 Feb 2026 20:54:33 +0300 Subject: [PATCH] =?UTF-8?q?feat(monitoring):=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=B4=D0=B0=D1=88=D0=B1=D0=BE=D1=80?= =?UTF-8?q?=D0=B4=20Grafana=20=D0=B4=D0=BB=D1=8F=20=D0=BC=D0=BE=D0=BD?= =?UTF-8?q?=D0=B8=D1=82=D0=BE=D1=80=D0=B8=D0=BD=D0=B3=D0=B0=20generator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Зачем: - нужна визуализация метрик generator в реальном времени - Prometheus job уже настроен, не хватает Grafana dashboard - Что: - добавлен provisioning-файл dashboards/generator-overview.json - 6 разделов: Overview, Events by Topic, Errors, Tick Statistics, Status, Info - Overview: Total Events/min (все 4 топика), Tick Duration (p50/p99) - Events by Topic: bar chart Events per Hour, Events Rate, Total Events - Errors: Total Errors, Error Rate, Errors by Topic (с 'or on() vector(0)') - Tick Statistics: Duration Distribution, Hour Factor (text), Tick Interval (text) - Status: Generator Status (threshold 120s), Generator Health (heartbeat), Time Since Last Tick - Info: команды и предупреждения о хардкоде GEN_TICK_SECONDS=5s - исправлены панели ошибок с 'or on() vector(0)' для корректного отображения 0 - заменен heatmap на bar chart для стабильности - добавлены пояснения про Events/min = сумма 4 связанных топиков - Hour Factor синхронизирован с кодом генератора (00-05/09-18) - Generator Health: переименовано из State Management с value mappings - обновлены docs/OPERATIONS.md и generator/README.md - удален устаревший plans/generator-monitoring-plan.md - Проверка: - дашборд открывается на http://localhost:3000/d/generator-overview - все панели отображают данные корректно (протестировано через Playwright) - ошибки показывают 0 вместо No data - Events/min корректно отображает сумму всех 4 топиков (~800-1000/min) --- .../dashboards/generator-overview.json | 1354 +++++++++++++++++ docs/OPERATIONS.md | 52 +- generator/README.md | 40 + 3 files changed, 1445 insertions(+), 1 deletion(-) create mode 100644 configs/grafana/provisioning/dashboards/generator-overview.json diff --git a/configs/grafana/provisioning/dashboards/generator-overview.json b/configs/grafana/provisioning/dashboards/generator-overview.json new file mode 100644 index 0000000..276635a --- /dev/null +++ b/configs/grafana/provisioning/dashboards/generator-overview.json @@ -0,0 +1,1354 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Обзор метрик Generator — автономного генератора событий для Kafka", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "evts/min" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum(rate(generator_events_total[1m])) * 60", + "refId": "A" + } + ], + "title": "Total Events/min (all 4 topics)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.99, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.50, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "median", + "refId": "B" + } + ], + "title": "Tick Duration (p50/p99)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "generator_last_success_timestamp * 1000", + "refId": "A" + } + ], + "title": "Last Successful Tick", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "panels": [], + "title": "Events by Topic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 6, + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum by (topic) (increase(generator_events_total[1h]))", + "format": "time_series", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Events per Hour by Topic", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "evts/min" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "rate(generator_events_total[1m]) * 60", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Events Rate by Topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "browser_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#5794F2" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "location_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#F2CC0C" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "device_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#73BF69" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "geo_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#B877D9" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 8, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum by (topic) (generator_events_total)", + "format": "table", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Total Events by Topic", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Count" + } + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 9, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum(generator_publish_errors_total) or on() vector(0)", + "refId": "A" + } + ], + "title": "Total Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "err/min" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum(rate(generator_publish_errors_total[5m])) * 60 or on() vector(0)", + "refId": "A" + } + ], + "title": "Error Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum by (topic) (rate(generator_publish_errors_total[5m]) * 60) or on() vector(0)", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Errors by Topic", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 13, + "panels": [], + "title": "Tick Statistics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.50, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.95, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.99, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Tick Duration Distribution", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 15, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**Hour Factor**\n\n🌙 Night (00-05): **0.7x**\n📈 Peak (09-18): **1.2x**\n☀️ Other: **1.0x**\n\n*Dashboard optimized for GEN_TICK_SECONDS=5s*", + "mode": "markdown" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "refId": "A" + } + ], + "title": "Hour Factor", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 16, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**Tick Interval**\n\n⏱️ **5 seconds**\n\n*Configurable via*\n`GEN_TICK_SECONDS`", + "mode": "markdown" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "refId": "A" + } + ], + "title": "Tick Interval", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "avg_over_time(sum(increase(generator_events_total[1m]))[5m:]) / 12", + "refId": "A" + } + ], + "title": "Events per Tick (avg)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 18, + "panels": [], + "title": "Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DISABLED" + }, + "1": { + "color": "green", + "index": 1, + "text": "ENABLED" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 19, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "(time() - generator_last_success_timestamp) \u003c bool 120", + "refId": "A" + } + ], + "title": "Generator Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DEGRADED" + }, + "1": { + "color": "green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "(time() - generator_last_success_timestamp) < bool 300", + "refId": "A" + } + ], + "title": "Generator Health (Heartbeat)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "time() - generator_last_success_timestamp", + "refId": "A" + } + ], + "title": "Time Since Last Tick", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 22, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**Generator Info**\n\n- **Topics:** browser_events, location_events, device_events, geo_events (4 linked topics)\n- **Events/min:** Total = sum of all 4 topics (each tick generates events across all topics)\n- **Tick Interval:** 5s (configurable via GEN_TICK_SECONDS)\n- **Target Rate:** ~200 events/min per topic × 4 topics = ~800 total events/min\n- **Hour Factor:** Night (00-05): 0.7x | Peak (09-18): 1.2x | Other: 1.0x\n- **State:** Saved to Kafka topic 'generator_state'\n- **History:** Batch history in 'generator_batch_history'\n- **⚠️ Note:** This dashboard is optimized for GEN_TICK_SECONDS=5s. Changing the tick interval will affect metrics accuracy (Events per Tick formula, Status thresholds). Generator Health shows "alive" status only (last successful tick), not state persistence. Check logs for state save errors.\n\n**Useful Commands:**\n```bash\n# View generator logs\nmake generator-logs\n\n# Check metrics endpoint\ncurl http://localhost:9109/metrics\n\n# Read batch history\ndocker compose exec kafka kafka-console-consumer \\\n --bootstrap-server localhost:9092 \\\n --topic generator_batch_history \\\n --from-beginning | head -20\n```", + "mode": "markdown" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "refId": "A" + } + ], + "title": "Info & Commands", + "type": "text" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "generator", + "kafka", + "monitoring" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Generator Overview", + "uid": "generator-overview", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index cd04e22..8c1b1a1 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -119,6 +119,56 @@ Prometheus метрики доступны на `http://localhost:9109/metrics`: - `generator_publish_errors_total` — ошибки публикации - `generator_tick_duration_seconds` — длительность тика +### Мониторинг через Grafana + +**Dashboard URL:** `http://localhost:3000/d/generator-overview` + +Дашборд "Generator Overview" автоматически загружается при старте Grafana и содержит: + +| Раздел | Панели | Описание | +|--------|--------|----------| +| **Overview** | Events/min | Скорость генерации событий в минуту | +| | Tick Duration | Медиана и p99 длительности тика | +| | Last Successful Tick | Время последнего успешного тика | +| **Events by Topic** | Events per Hour (24h bar chart) | Распределение событий по часам и топикам | +| | Events Rate by Topic | График по 4 топикам (browser, location, device, geo) | +| | Total Events by Topic | Суммарные счётчики по каждому топику | +| **Errors** | Total Errors | Общее число ошибок публикации | +| | Error Rate | Скорость ошибок (err/min) | +| | Errors by Topic | Ошибки разбиты по топикам | +| **Tick Statistics** | Tick Duration Distribution | p50, p95, p99 длительности тиков | +| | Events per Tick | Среднее число событий на тик | +| | Hour Factor | Текущий временной множитель (0.7/1.0/1.2) | +| **Status** | Generator Status | Статус работы (enabled/disabled) | +| | Generator Health | Статус активности (heartbeat last tick), не проверяет state save | +| | Time Since Last Tick | Время с последнего тика | +| **Info** | Полезные команды и параметры конфигурации | + +#### Troubleshooting генератора + +**Нет данных на дашборде:** +1. Проверить, что генератор запущен: `docker compose ps generator` +2. Проверить метрики напрямую: `curl http://localhost:9109/metrics` +3. Проверить target в Prometheus: `http://localhost:9090/targets` (job: generator) + +**Высокий error rate:** +- Проверить доступность Kafka: `docker compose ps kafka` +- Смотреть логи: `make generator-logs` +- Проверить consumer lag: дашборд Kafka Overview + +**Длительные тики (p99 > 1s):** +- Проверить CPU/ресурсы контейнера +- Возможно, высокая нагрузка на Kafka — проверить дашборд Kafka + +**Dashboard не загрузился:** +```bash +# Перезагрузить provisioning Grafana +curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload + +# Или пересоздать контейнер +docker compose restart grafana +``` + ## Рекомендуемый сценарий (фаза 2) ```bash @@ -190,7 +240,7 @@ curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"' - Airflow отправляет метрики в StatsD-формате на `statsd-exporter:8125` - Mapping конфигурация: `configs/statsd_mapping.yml` - **Grafana provisioning** (`configs/grafana/provisioning/`): - - Дашборды: ClickHouse Overview, Kafka Overview, Airflow Overview + - Дашборды: ClickHouse Overview, Kafka Overview, Airflow Overview, Generator Overview - Алерты: ClickHouse, Kafka, Airflow ### После `git pull`: быстрый апдейт мониторинга diff --git a/generator/README.md b/generator/README.md index f962d96..31185ca 100644 --- a/generator/README.md +++ b/generator/README.md @@ -85,6 +85,46 @@ curl http://localhost:9109/metrics curl http://localhost:9090/api/v1/targets | grep generator ``` +## Мониторинг в Grafana + +**Dashboard URL:** `http://localhost:3000/d/generator-overview` + +Дашборд "Generator Overview" предоставляет полную визуализацию работы генератора: + +### Ключевые панели + +| Панель | Метрика | Описание | +|--------|---------|----------| +| **Events/min** | `rate(generator_events_total[1m]) * 60` | Текущая скорость генерации | +| **Tick Duration** | `generator_tick_duration_seconds` | p50 и p99 длительности тика | +| **Last Successful Tick** | `generator_last_success_timestamp` | Время последнего успешного тика | +| **Events per Hour** | `increase(generator_events_total[1h])` | 24-часовое распределение по топикам (bar chart) | +| **Errors** | `generator_publish_errors_total` | Общее число и rate ошибок | +| **Generator Status** | derived | Активен ли генератор | + +### Структура дашборда + +Дашборд разделён на 6 секций: + +1. **Overview** — ключевые метрики (events/min, tick duration, last success) +2. **Events by Topic** — bar chart Events per Hour, rate by topic, total counters +3. **Errors** — total errors, error rate, errors by topic +4. **Tick Statistics** — duration distribution (p50/p95/p99), events per tick, hour factor +5. **Status** — generator status, generator health (heartbeat), time since last tick +6. **Info** — полезные команды и параметры конфигурации + +### Доступ к дашборду + +Дашборд автоматически загружается в Grafana при старте контейнера (provisioning). + +```bash +# Открыть дашборд +open http://localhost:3000/d/generator-overview + +# Перезагрузить provisioning (если дашборд не появился) +curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload +``` + ## История batch История пишется в Kafka-топик `generator_batch_history` (JSON).