diff --git a/configs/grafana/provisioning/dashboards/generator-overview.json b/configs/grafana/provisioning/dashboards/generator-overview.json new file mode 100644 index 0000000..276635a --- /dev/null +++ b/configs/grafana/provisioning/dashboards/generator-overview.json @@ -0,0 +1,1354 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Обзор метрик Generator — автономного генератора событий для Kafka", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "evts/min" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum(rate(generator_events_total[1m])) * 60", + "refId": "A" + } + ], + "title": "Total Events/min (all 4 topics)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.99, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.50, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "median", + "refId": "B" + } + ], + "title": "Tick Duration (p50/p99)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "generator_last_success_timestamp * 1000", + "refId": "A" + } + ], + "title": "Last Successful Tick", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "panels": [], + "title": "Events by Topic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 6, + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum by (topic) (increase(generator_events_total[1h]))", + "format": "time_series", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Events per Hour by Topic", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "evts/min" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "rate(generator_events_total[1m]) * 60", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Events Rate by Topic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "browser_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#5794F2" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "location_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#F2CC0C" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "device_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#73BF69" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "geo_events" + }, + "properties": [ + { + "id": "color", + "value": { + "fixed": "#B877D9" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 8, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum by (topic) (generator_events_total)", + "format": "table", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Total Events by Topic", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Count" + } + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 9, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum(generator_publish_errors_total) or on() vector(0)", + "refId": "A" + } + ], + "title": "Total Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "err/min" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum(rate(generator_publish_errors_total[5m])) * 60 or on() vector(0)", + "refId": "A" + } + ], + "title": "Error Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "sum by (topic) (rate(generator_publish_errors_total[5m]) * 60) or on() vector(0)", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Errors by Topic", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 13, + "panels": [], + "title": "Tick Statistics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.50, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.95, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "histogram_quantile(0.99, sum(rate(generator_tick_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Tick Duration Distribution", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 15, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**Hour Factor**\n\n🌙 Night (00-05): **0.7x**\n📈 Peak (09-18): **1.2x**\n☀️ Other: **1.0x**\n\n*Dashboard optimized for GEN_TICK_SECONDS=5s*", + "mode": "markdown" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "refId": "A" + } + ], + "title": "Hour Factor", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 16, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**Tick Interval**\n\n⏱️ **5 seconds**\n\n*Configurable via*\n`GEN_TICK_SECONDS`", + "mode": "markdown" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "refId": "A" + } + ], + "title": "Tick Interval", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "avg_over_time(sum(increase(generator_events_total[1m]))[5m:]) / 12", + "refId": "A" + } + ], + "title": "Events per Tick (avg)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 18, + "panels": [], + "title": "Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DISABLED" + }, + "1": { + "color": "green", + "index": 1, + "text": "ENABLED" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 19, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "(time() - generator_last_success_timestamp) \u003c bool 120", + "refId": "A" + } + ], + "title": "Generator Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "DEGRADED" + }, + "1": { + "color": "green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 20, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "(time() - generator_last_success_timestamp) < bool 300", + "refId": "A" + } + ], + "title": "Generator Health (Heartbeat)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "expr": "time() - generator_last_success_timestamp", + "refId": "A" + } + ], + "title": "Time Since Last Tick", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 22, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**Generator Info**\n\n- **Topics:** browser_events, location_events, device_events, geo_events (4 linked topics)\n- **Events/min:** Total = sum of all 4 topics (each tick generates events across all topics)\n- **Tick Interval:** 5s (configurable via GEN_TICK_SECONDS)\n- **Target Rate:** ~200 events/min per topic × 4 topics = ~800 total events/min\n- **Hour Factor:** Night (00-05): 0.7x | Peak (09-18): 1.2x | Other: 1.0x\n- **State:** Saved to Kafka topic 'generator_state'\n- **History:** Batch history in 'generator_batch_history'\n- **⚠️ Note:** This dashboard is optimized for GEN_TICK_SECONDS=5s. Changing the tick interval will affect metrics accuracy (Events per Tick formula, Status thresholds). Generator Health shows "alive" status only (last successful tick), not state persistence. Check logs for state save errors.\n\n**Useful Commands:**\n```bash\n# View generator logs\nmake generator-logs\n\n# Check metrics endpoint\ncurl http://localhost:9109/metrics\n\n# Read batch history\ndocker compose exec kafka kafka-console-consumer \\\n --bootstrap-server localhost:9092 \\\n --topic generator_batch_history \\\n --from-beginning | head -20\n```", + "mode": "markdown" + }, + "pluginVersion": "11.0.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus_uid" + }, + "refId": "A" + } + ], + "title": "Info & Commands", + "type": "text" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "generator", + "kafka", + "monitoring" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Generator Overview", + "uid": "generator-overview", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index cd04e22..8c1b1a1 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -119,6 +119,56 @@ Prometheus метрики доступны на `http://localhost:9109/metrics`: - `generator_publish_errors_total` — ошибки публикации - `generator_tick_duration_seconds` — длительность тика +### Мониторинг через Grafana + +**Dashboard URL:** `http://localhost:3000/d/generator-overview` + +Дашборд "Generator Overview" автоматически загружается при старте Grafana и содержит: + +| Раздел | Панели | Описание | +|--------|--------|----------| +| **Overview** | Events/min | Скорость генерации событий в минуту | +| | Tick Duration | Медиана и p99 длительности тика | +| | Last Successful Tick | Время последнего успешного тика | +| **Events by Topic** | Events per Hour (24h bar chart) | Распределение событий по часам и топикам | +| | Events Rate by Topic | График по 4 топикам (browser, location, device, geo) | +| | Total Events by Topic | Суммарные счётчики по каждому топику | +| **Errors** | Total Errors | Общее число ошибок публикации | +| | Error Rate | Скорость ошибок (err/min) | +| | Errors by Topic | Ошибки разбиты по топикам | +| **Tick Statistics** | Tick Duration Distribution | p50, p95, p99 длительности тиков | +| | Events per Tick | Среднее число событий на тик | +| | Hour Factor | Текущий временной множитель (0.7/1.0/1.2) | +| **Status** | Generator Status | Статус работы (enabled/disabled) | +| | Generator Health | Статус активности (heartbeat last tick), не проверяет state save | +| | Time Since Last Tick | Время с последнего тика | +| **Info** | Полезные команды и параметры конфигурации | + +#### Troubleshooting генератора + +**Нет данных на дашборде:** +1. Проверить, что генератор запущен: `docker compose ps generator` +2. Проверить метрики напрямую: `curl http://localhost:9109/metrics` +3. Проверить target в Prometheus: `http://localhost:9090/targets` (job: generator) + +**Высокий error rate:** +- Проверить доступность Kafka: `docker compose ps kafka` +- Смотреть логи: `make generator-logs` +- Проверить consumer lag: дашборд Kafka Overview + +**Длительные тики (p99 > 1s):** +- Проверить CPU/ресурсы контейнера +- Возможно, высокая нагрузка на Kafka — проверить дашборд Kafka + +**Dashboard не загрузился:** +```bash +# Перезагрузить provisioning Grafana +curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload + +# Или пересоздать контейнер +docker compose restart grafana +``` + ## Рекомендуемый сценарий (фаза 2) ```bash @@ -190,7 +240,7 @@ curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"' - Airflow отправляет метрики в StatsD-формате на `statsd-exporter:8125` - Mapping конфигурация: `configs/statsd_mapping.yml` - **Grafana provisioning** (`configs/grafana/provisioning/`): - - Дашборды: ClickHouse Overview, Kafka Overview, Airflow Overview + - Дашборды: ClickHouse Overview, Kafka Overview, Airflow Overview, Generator Overview - Алерты: ClickHouse, Kafka, Airflow ### После `git pull`: быстрый апдейт мониторинга diff --git a/generator/README.md b/generator/README.md index f962d96..31185ca 100644 --- a/generator/README.md +++ b/generator/README.md @@ -85,6 +85,46 @@ curl http://localhost:9109/metrics curl http://localhost:9090/api/v1/targets | grep generator ``` +## Мониторинг в Grafana + +**Dashboard URL:** `http://localhost:3000/d/generator-overview` + +Дашборд "Generator Overview" предоставляет полную визуализацию работы генератора: + +### Ключевые панели + +| Панель | Метрика | Описание | +|--------|---------|----------| +| **Events/min** | `rate(generator_events_total[1m]) * 60` | Текущая скорость генерации | +| **Tick Duration** | `generator_tick_duration_seconds` | p50 и p99 длительности тика | +| **Last Successful Tick** | `generator_last_success_timestamp` | Время последнего успешного тика | +| **Events per Hour** | `increase(generator_events_total[1h])` | 24-часовое распределение по топикам (bar chart) | +| **Errors** | `generator_publish_errors_total` | Общее число и rate ошибок | +| **Generator Status** | derived | Активен ли генератор | + +### Структура дашборда + +Дашборд разделён на 6 секций: + +1. **Overview** — ключевые метрики (events/min, tick duration, last success) +2. **Events by Topic** — bar chart Events per Hour, rate by topic, total counters +3. **Errors** — total errors, error rate, errors by topic +4. **Tick Statistics** — duration distribution (p50/p95/p99), events per tick, hour factor +5. **Status** — generator status, generator health (heartbeat), time since last tick +6. **Info** — полезные команды и параметры конфигурации + +### Доступ к дашборду + +Дашборд автоматически загружается в Grafana при старте контейнера (provisioning). + +```bash +# Открыть дашборд +open http://localhost:3000/d/generator-overview + +# Перезагрузить provisioning (если дашборд не появился) +curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload +``` + ## История batch История пишется в Kafka-топик `generator_batch_history` (JSON).