From e851ef97885ed1708e86c83c64c24833d788131e Mon Sep 17 00:00:00 2001
From: Dmitry Dementev <me@dementev.space>
Date: Mon, 9 Feb 2026 09:28:13 +0300
Subject: [PATCH] fix(monitoring): add recover flow for stuck monitoring stack

- Why:
  - during intensive development monitoring can get stuck (No data, out of bounds)
  - regular reload is not always enough to recover Prometheus + StatsD pipeline
- What:
  - add make target recover-monitoring for hard recovery path
  - recreate prometheus and statsd-exporter, restart airflow scheduler/webserver
  - keep Grafana provisioning reload and target checks in one command
  - document when to use recover-monitoring in OPERATIONS runbook
- Check:
  - run make recover-monitoring
  - verify Prometheus targets for airflow/clickhouse/kafka are up
---
 Makefile           | 17 ++++++++++++++++-
 docs/OPERATIONS.md | 10 ++++++++--
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 2b3de66..ec600e8 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: up ddl data transform reload-monitoring
+.PHONY: up ddl data transform reload-monitoring recover-monitoring
 
 COMPOSE ?= docker compose
 
@@ -26,3 +26,18 @@ reload-monitoring:
 	@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload && echo " [alerting]"
 	@echo "=== Проверка ==="
 	@curl -s http://localhost:9090/api/v1/targets | grep -o '"job":"[^"]*"' | sort | uniq
+
+# Восстановление мониторинга после сбоев (например, out of bounds / пустые дашборды)
+recover-monitoring:
+	@echo "=== Восстановление мониторинга (жесткий режим) ==="
+	$(COMPOSE) rm -sf prometheus statsd-exporter
+	$(COMPOSE) up -d prometheus grafana kafka-exporter statsd-exporter
+	$(COMPOSE) restart airflow-scheduler airflow-webserver
+	@echo "=== Перезагрузка provisioning Grafana ==="
+	@sleep 2
+	@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload && echo " [datasources]"
+	@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload && echo " [dashboards]"
+	@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload && echo " [alerting]"
+	@echo "=== Проверка targets ==="
+	@curl -s http://localhost:9090/api/v1/targets | grep -o '"job":"[^"]*"' | sort | uniq
+	@curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"' | sort | uniq -c
diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md
index b5ffe85..efcc33d 100644
--- a/docs/OPERATIONS.md
+++ b/docs/OPERATIONS.md
@@ -95,6 +95,8 @@ docker compose exec -T clickhouse clickhouse-client --user=default --password=12
 ```bash
 # Быстрый вариант (make)
 make reload-monitoring
+# Если мониторинг "залип" (No data/out of bounds) — жесткое восстановление:
+make recover-monitoring
 
 # Или вручную:
 docker compose up -d prometheus grafana kafka-exporter statsd-exporter
@@ -136,10 +138,13 @@ curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"'
 ```bash
 # Рекомендуемый способ (через make)
 make reload-monitoring
+# Если метрики пропали/залипли:
+make recover-monitoring
 
 # Или вручную:
-docker compose up -d prometheus grafana kafka-exporter
-docker compose restart prometheus
+docker compose rm -sf prometheus statsd-exporter
+docker compose up -d prometheus grafana kafka-exporter statsd-exporter
+docker compose restart airflow-scheduler airflow-webserver
 curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload
 curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload
 curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload
@@ -248,6 +253,7 @@ curl -s -X POST -u admin:admin http://localhost:3000/api/admin/provisioning/aler
 **Общие проблемы:**
 - **"No data" в Grafana**: проверить, что Prometheus видит target (`Status -> Targets` в UI)
 - **Dashboard не загрузился**: проверить логи Grafana — provisioning работает при первом старте контейнера
+- **Prometheus spam `out of bounds` и дашборды пустые**: выполнить `make recover-monitoring`
 
 **ClickHouse:**
 - **Метрики не обновляются**: ClickHouse экспортирует метрики на `0.0.0.0:9126` внутри сети Docker