apiVersion: 1 groups: - orgId: 1 name: airflow_health_group folder: Airflow Alerts interval: 30s rules: # --- Alert: Airflow Scheduler Down --- - uid: airflow_scheduler_down title: Airflow Scheduler Down condition: C data: - refId: A relativeTimeRange: from: 300 to: 0 datasourceUid: prometheus_uid model: datasource: type: prometheus uid: prometheus_uid editorMode: code expr: rate(airflow_scheduler_heartbeat_total[5m]) instant: false intervalMs: 1000 legendFormat: __auto maxDataPoints: 43200 range: true refId: A - refId: B datasourceUid: __expr__ model: conditions: - evaluator: params: - 0.1 type: lt operator: type: and query: params: - B reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: A intervalMs: 1000 maxDataPoints: 43200 refId: B type: threshold - refId: C datasourceUid: __expr__ model: conditions: - evaluator: params: - 0 type: gt operator: type: and query: params: - C reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: B intervalMs: 1000 maxDataPoints: 43200 refId: C type: threshold noDataState: Alerting execErrState: Error for: 2m annotations: summary: "Airflow scheduler не отправляет heartbeats" description: "Scheduler possible down — rate(airflow_scheduler_heartbeat_total) < 0.1 в течение 2 минут" labels: severity: critical # --- Alert: Airflow Queue Backlog --- - uid: airflow_queue_high title: Airflow Queue Backlog condition: C data: - refId: A relativeTimeRange: from: 300 to: 0 datasourceUid: prometheus_uid model: datasource: type: prometheus uid: prometheus_uid editorMode: code expr: airflow_executor_queued_tasks instant: false intervalMs: 1000 legendFormat: __auto maxDataPoints: 43200 range: true refId: A - refId: B datasourceUid: __expr__ model: conditions: - evaluator: params: - 50 type: gt operator: type: and query: params: - B reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: A intervalMs: 1000 maxDataPoints: 43200 refId: B type: threshold - refId: C datasourceUid: __expr__ model: conditions: - evaluator: params: - 0 type: gt operator: type: and query: params: - C reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: B intervalMs: 1000 maxDataPoints: 43200 refId: C type: threshold noDataState: OK execErrState: Error for: 5m annotations: summary: "В очереди Airflow > 50 задач" description: "airflow_executor_queued_tasks = {{ $values.A }} (превышает 50 в течение 5 минут) — возможна перегрузка" labels: severity: warning # --- Alert: High Task Failure Rate --- - uid: airflow_task_failures_high title: High Task Failure Rate condition: C data: - refId: A relativeTimeRange: from: 300 to: 0 datasourceUid: prometheus_uid model: datasource: type: prometheus uid: prometheus_uid editorMode: code expr: rate(airflow_task_failures_total[5m]) instant: false intervalMs: 1000 legendFormat: __auto maxDataPoints: 43200 range: true refId: A - refId: B datasourceUid: __expr__ model: conditions: - evaluator: params: - 0.1 type: gt operator: type: and query: params: - B reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: A intervalMs: 1000 maxDataPoints: 43200 refId: B type: threshold - refId: C datasourceUid: __expr__ model: conditions: - evaluator: params: - 0 type: gt operator: type: and query: params: - C reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: B intervalMs: 1000 maxDataPoints: 43200 refId: C type: threshold noDataState: OK execErrState: Error for: 3m annotations: summary: "Высокий rate падений тасков" description: "rate(airflow_task_failures_total) = {{ $values.A }} (превышает 0.1 в течение 3 минут) — проверьте логи DAG" labels: severity: warning # --- Alert: High DAG Parse Time --- - uid: airflow_parse_time_high title: High DAG Parse Time condition: C data: - refId: A relativeTimeRange: from: 300 to: 0 datasourceUid: prometheus_uid model: datasource: type: prometheus uid: prometheus_uid editorMode: code expr: airflow_dag_processing_total_parse_time instant: false intervalMs: 1000 legendFormat: __auto maxDataPoints: 43200 range: true refId: A - refId: B datasourceUid: __expr__ model: conditions: - evaluator: params: - 30 type: gt operator: type: and query: params: - B reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: A intervalMs: 1000 maxDataPoints: 43200 refId: B type: threshold - refId: C datasourceUid: __expr__ model: conditions: - evaluator: params: - 0 type: gt operator: type: and query: params: - C reducer: params: [] type: last type: query datasource: type: __expr__ uid: __expr__ expression: B intervalMs: 1000 maxDataPoints: 43200 refId: C type: threshold noDataState: OK execErrState: Error for: 5m annotations: summary: "Парсинг DAG занимает > 30 секунд" description: "airflow_dag_processing_total_parse_time = {{ $values.A }}s — возможно, есть тяжелые DAG" labels: severity: info