fix(monitoring): align airflow statsd mapping and reload flow

- Why:
  - Airflow task metrics were mapped to non-emitted StatsD keys
  - reload-monitoring did not restart statsd-exporter after mapping changes
- What:
  - update StatsD mapping for Airflow 2.10.5 metric names
  - remove problematic catch-all mapping that produced inconsistent series
  - restart statsd-exporter in reload-monitoring flow
  - sync operations runbook and airflow monitoring plan with actual metrics
- Check:
  - make reload-monitoring
  - Prometheus targets: airflow/clickhouse/kafka are UP
  - trigger ddl_init and verify airflow_task_duration_seconds_count
  - verify airflow_task_success_total and airflow_task_failures_total in Prometheus

Makefile

@@ -18,7 +18,7 @@ transform:
 reload-monitoring:
 	@echo "=== Перезагрузка сервисов мониторинга ==="
 	$(COMPOSE) up -d prometheus grafana kafka-exporter statsd-exporter
-	$(COMPOSE) restart prometheus
+	$(COMPOSE) restart prometheus statsd-exporter
 	@echo "=== Перезагрузка provisioning Grafana ==="
 	@sleep 2
 	@curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload && echo " [datasources]"

configs/statsd_mapping.yml

@@ -41,7 +41,8 @@ mappings:
     type: gauge
 
   # Task duration (timer → _seconds суффикс)
-  - match: "airflow.dag.*.task.*.duration"
+  # Проверено для Airflow 2.10.5: metric key = dag.<dag_id>.<task_id>.duration
+  - match: "airflow.dag.*.*.duration"
     name: "airflow_task_duration_seconds"
     help: "Task duration"
     type: timer
@@ -49,26 +50,20 @@ mappings:
       dag_id: "$1"
       task_id: "$2"
 
-  # Task failures (counter → _total суффикс)
+  # Task failures/success (counter → _total суффикс)
-  - match: "airflow.dag.*.task.*.failures"
+  # Проверено для Airflow 2.10.5: metric key = ti.finish.<dag_id>.<task_id>.<state>
+  - match: "airflow.ti.finish.*.*.failed"
     name: "airflow_task_failures_total"
-    help: "Task failures"
+    help: "Task failures by dag_id and task_id"
     type: counter
     labels:
       dag_id: "$1"
       task_id: "$2"
 
-  # Task success (counter → _total суффикс)
+  - match: "airflow.ti.finish.*.*.success"
-  - match: "airflow.dag.*.task.*.success"
     name: "airflow_task_success_total"
-    help: "Task success"
+    help: "Task successes by dag_id and task_id"
     type: counter
     labels:
       dag_id: "$1"
       task_id: "$2"
-
-  # Catch-all для остальных airflow метрик
-  - match: "airflow.*.*"
-    name: "airflow_${1}_${2}"
-    help: "Airflow metric $1.$2"
-    type: gauge

docs/OPERATIONS.md

@@ -98,7 +98,7 @@ make reload-monitoring
 
 # Или вручную:
 docker compose up -d prometheus grafana kafka-exporter statsd-exporter
-docker compose restart prometheus
+docker compose restart prometheus statsd-exporter
 curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/datasources/reload
 curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload
 curl -s -u admin:admin -X POST http://localhost:3000/api/admin/provisioning/alerting/reload

plans/monitoring_airflow_plan.md

@@ -16,10 +16,10 @@
 - `airflow_executor_open_slots` — доступные слоты executor
 - `airflow_executor_queued_tasks` — задачи в очереди
 - `airflow_executor_running_tasks` — запущенные задачи
-- `airflow_task_duration{task_id, dag_id}` — длительность выполнения тасков
+- `airflow_task_duration_seconds{task_id, dag_id}` — длительность выполнения тасков
-- `airflow_task_failures` — количество падений тасков
+- `airflow_task_failures_total{task_id, dag_id}` — количество падений тасков
-- `airflow_task_success` — успешные выполнения
+- `airflow_task_success_total{task_id, dag_id}` — успешные выполнения
-- `airflow_scheduler_heartbeat` — heartbeats шедулера
+- `airflow_scheduler_heartbeat_total` — heartbeats шедулера
 
 ---
 
@@ -114,8 +114,9 @@ mappings:
       operator: "$1"
 
   # Task duration by dag_id и task_id
-  - match: "airflow.dag.*.task.*.duration"
+  # Проверено на Airflow 2.10.5: dag.<dag_id>.<task_id>.duration
-    name: "airflow_task_duration"
+  - match: "airflow.dag.*.*.duration"
+    name: "airflow_task_duration_seconds"
     help: "Task duration by dag_id and task_id"
     type: timer
     labels:
@@ -123,16 +124,17 @@ mappings:
       task_id: "$2"
 
   # Task failures/success by dag_id и task_id
-  - match: "airflow.dag.*.task.*.failures"
+  # Проверено на Airflow 2.10.5: ti.finish.<dag_id>.<task_id>.<state>
-    name: "airflow_task_failures"
+  - match: "airflow.ti.finish.*.*.failed"
+    name: "airflow_task_failures_total"
     help: "Task failures by dag_id and task_id"
     type: counter
     labels:
       dag_id: "$1"
       task_id: "$2"
 
-  - match: "airflow.dag.*.task.*.success"
+  - match: "airflow.ti.finish.*.*.success"
-    name: "airflow_task_success"
+    name: "airflow_task_success_total"
     help: "Task success by dag_id and task_id"
     type: counter
     labels:
@@ -141,7 +143,7 @@ mappings:
 
   # Scheduler metrics
   - match: "airflow.scheduler_heartbeat"
-    name: "airflow_scheduler_heartbeat"
+    name: "airflow_scheduler_heartbeat_total"
     help: "Scheduler heartbeats"
     type: counter
 
@@ -150,11 +152,6 @@ mappings:
     help: "Scheduler critical section busy"
     type: gauge
 
-  # Catch-all для остальных airflow метрик
-  - match: "airflow.*"
-    name: "airflow_${1}"
-    help: "Airflow metric $1"
-    type: gauge
 ```
 
 ### 2.3. configs/prometheus.yml
@@ -204,9 +201,9 @@ scrape_configs:
 | Executor Slots | `airflow_executor_open_slots` | Доступные слоты |
 | Queued Tasks | `airflow_executor_queued_tasks` | Задачи в очереди |
 | Running Tasks | `airflow_executor_running_tasks` | Выполняемые задачи |
-| Task Duration | `rate(airflow_task_duration_sum[5m]) / rate(airflow_task_duration_count[5m])` | Средняя длительность тасков |
+| Task Duration | `rate(airflow_task_duration_seconds_sum[5m]) / rate(airflow_task_duration_seconds_count[5m])` | Средняя длительность тасков |
-| Task Failures | `rate(airflow_task_failures[5m])` | Rate падений по таскам |
+| Task Failures | `rate(airflow_task_failures_total[5m])` | Rate падений по таскам |
-| Scheduler Heartbeat | `rate(airflow_scheduler_heartbeat[5m])` | Активность шедулера |
+| Scheduler Heartbeat | `rate(airflow_scheduler_heartbeat_total[5m])` | Активность шедулера |
 
 ### 3.2. Структура дашборда (основные секции)
 
@@ -270,7 +267,7 @@ groups:
               to: 0
             datasourceUid: prometheus_uid
             model:
-              expr: rate(airflow_scheduler_heartbeat[5m])
+              expr: rate(airflow_scheduler_heartbeat_total[5m])
               instant: true
           - refId: B
             relativeTimeRange:
@@ -289,7 +286,7 @@ groups:
         for: 2m
         annotations:
           summary: "Airflow scheduler не отправляет heartbeats"
-          description: "Scheduler possible down — rate(airflow_scheduler_heartbeat) < 0.1 в течение 2 минут"
+          description: "Scheduler possible down — rate(airflow_scheduler_heartbeat_total) < 0.1 в течение 2 минут"
         labels:
           severity: critical
 
@@ -329,7 +326,7 @@ groups:
           - refId: A
             datasourceUid: prometheus_uid
             model:
-              expr: rate(airflow_task_failures[5m])
+              expr: rate(airflow_task_failures_total[5m])
               instant: true
           - refId: B
             datasourceUid: __expr__
@@ -345,7 +342,7 @@ groups:
         for: 3m
         annotations:
           summary: "Высокий rate падений тасков"
-          description: "rate(airflow_task_failures) > 0.1 в течение 3 минут — проверьте логи DAG"
+          description: "rate(airflow_task_failures_total) > 0.1 в течение 3 минут — проверьте логи DAG"
         labels:
           severity: warning