fix(dq): устранено переполнение памяти проверок

- Зачем:
  - параллельный DQ выгружал деловые ключи в Python и исчерпывал 640 МиБ планировщика.
- Что:
  - четыре сверки агрегируют дневные итоги и ограниченную диагностику в ClickHouse.
  - коробка планировщика поднята до 1,5 ГиБ, локальные пределы задач удалены.
  - измерения и граница между SQL и Python записаны в документации.
- Проверка:
  - make lint, make config-test, make smoke, make check-clickhouse, make check-services.
  - полный etl_pipeline, красный и восстановительный прогоны dq_check.

sql/dq/daily_traffic_vs_dds.sql

@@ -23,24 +23,49 @@ WITH
             any(known_users) AS known_users
         FROM dm.daily_traffic_v
         GROUP BY report_date
+    ),
+    comparison AS
+    (
+        SELECT
+            coalesce(reference.report_date, actual.report_date) AS data_date,
+            toString(coalesce(reference.report_date, actual.report_date))
+                AS business_key,
+            isNotNull(reference.report_date) AS reference_present,
+            isNotNull(actual.report_date) AS actual_present,
+            toString(tuple(reference.visitors, reference.known_users))
+                AS reference_value,
+            concat(
+                toString(tuple(actual.visitors, actual.known_users)),
+                ', rows=',
+                toString(actual.physical_rows)
+            ) AS actual_value,
+            isNull(reference.report_date)
+                OR isNull(actual.report_date)
+                OR actual.physical_rows != 1
+                OR NOT isNotDistinctFrom(reference.visitors, actual.visitors)
+                OR NOT isNotDistinctFrom(reference.known_users, actual.known_users)
+                AS failed
+        FROM reference
+        GLOBAL FULL OUTER JOIN actual USING (report_date)
     )
 SELECT
-    coalesce(reference.report_date, actual.report_date) AS data_date,
-    toString(coalesce(reference.report_date, actual.report_date)) AS business_key,
-    isNotNull(reference.report_date) AS reference_present,
-    isNotNull(actual.report_date) AS actual_present,
-    toString(tuple(reference.visitors, reference.known_users)) AS reference_value,
-    concat(
-        toString(tuple(actual.visitors, actual.known_users)),
-        ', rows=',
-        toString(actual.physical_rows)
-    ) AS actual_value,
-    isNull(reference.report_date)
-        OR isNull(actual.report_date)
-        OR actual.physical_rows != 1
-        OR NOT isNotDistinctFrom(reference.visitors, actual.visitors)
-        OR NOT isNotDistinctFrom(reference.known_users, actual.known_users) AS failed
-FROM reference
-GLOBAL FULL OUTER JOIN actual USING (report_date)
+    data_date,
+    countIf(reference_present) AS reference_rows,
+    countIf(actual_present) AS actual_rows,
+    countIf(failed) AS failed_rows,
+    groupArrayIf(20)(
+        concat(
+            toString(data_date),
+            ' key=',
+            business_key,
+            ': эталон=',
+            if(reference_present, reference_value, 'нет ключа'),
+            ', объект=',
+            if(actual_present, actual_value, 'нет ключа')
+        ),
+        failed
+    ) AS diagnostics
+FROM comparison
+GROUP BY data_date
 ORDER BY data_date
 SETTINGS join_use_nulls = 1

sql/dq/purchase_vs_orders_vs_dds.sql

@@ -57,39 +57,64 @@ WITH
         GROUP BY
             order_day,
             order_id
+    ),
+    comparison AS
+    (
+        SELECT
+            coalesce(reference.order_day, actual.order_day) AS data_date,
+            coalesce(reference.order_id, actual.order_id) AS business_key,
+            isNotNull(reference.order_id) AS reference_present,
+            isNotNull(actual.order_id) AS actual_present,
+            toString(tuple(
+                reference.declared_revenue,
+                reference.items_total,
+                reference.status,
+                reference.mismatch_class
+            )) AS reference_value,
+            concat(
+                toString(tuple(
+                    actual.declared_revenue,
+                    actual.items_total,
+                    actual.status,
+                    actual.mismatch_class
+                )),
+                ', rows=',
+                toString(actual.physical_rows)
+            ) AS actual_value,
+            isNull(reference.order_id)
+                OR isNull(actual.order_id)
+                OR actual.physical_rows != 1
+                OR NOT isNotDistinctFrom(
+                    reference.declared_revenue,
+                    actual.declared_revenue
+                )
+                OR NOT isNotDistinctFrom(reference.items_total, actual.items_total)
+                OR NOT isNotDistinctFrom(reference.status, actual.status)
+                OR NOT isNotDistinctFrom(
+                    reference.mismatch_class,
+                    actual.mismatch_class
+                ) AS failed
+        FROM reference
+        GLOBAL FULL OUTER JOIN actual USING (order_day, order_id)
     )
 SELECT
-    coalesce(reference.order_day, actual.order_day) AS data_date,
-    coalesce(reference.order_id, actual.order_id) AS business_key,
-    isNotNull(reference.order_id) AS reference_present,
-    isNotNull(actual.order_id) AS actual_present,
-    toString(tuple(
-        reference.declared_revenue,
-        reference.items_total,
-        reference.status,
-        reference.mismatch_class
-    )) AS reference_value,
-    concat(
-        toString(tuple(
-            actual.declared_revenue,
-            actual.items_total,
-            actual.status,
-            actual.mismatch_class
-        )),
-        ', rows=',
-        toString(actual.physical_rows)
-    ) AS actual_value,
-    isNull(reference.order_id)
-        OR isNull(actual.order_id)
-        OR actual.physical_rows != 1
-        OR NOT isNotDistinctFrom(reference.declared_revenue, actual.declared_revenue)
-        OR NOT isNotDistinctFrom(reference.items_total, actual.items_total)
-        OR NOT isNotDistinctFrom(reference.status, actual.status)
-        OR NOT isNotDistinctFrom(reference.mismatch_class, actual.mismatch_class)
-        AS failed
-FROM reference
-GLOBAL FULL OUTER JOIN actual USING (order_day, order_id)
-ORDER BY
     data_date,
-    business_key
+    countIf(reference_present) AS reference_rows,
+    countIf(actual_present) AS actual_rows,
+    countIf(failed) AS failed_rows,
+    groupArrayIf(20)(
+        concat(
+            toString(data_date),
+            ' key=',
+            business_key,
+            ': эталон=',
+            if(reference_present, reference_value, 'нет ключа'),
+            ', объект=',
+            if(actual_present, actual_value, 'нет ключа')
+        ),
+        failed
+    ) AS diagnostics
+FROM comparison
+GROUP BY data_date
+ORDER BY data_date
 SETTINGS join_use_nulls = 1

sql/dq/revenue_daily_vs_dds.sql

@@ -49,29 +49,59 @@ WITH
         GROUP BY
             report_date,
             product_category
+    ),
+    comparison AS
+    (
+        SELECT
+            coalesce(reference.report_date, actual.report_date) AS data_date,
+            coalesce(reference.product_category, actual.product_category)
+                AS business_key,
+            isNotNull(reference.product_category) AS reference_present,
+            isNotNull(actual.product_category) AS actual_present,
+            toString(tuple(
+                reference.orders,
+                reference.units,
+                reference.revenue,
+                reference.aov
+            )) AS reference_value,
+            concat(
+                toString(tuple(
+                    actual.orders,
+                    actual.units,
+                    actual.revenue,
+                    actual.aov
+                )),
+                ', rows=',
+                toString(actual.physical_rows)
+            ) AS actual_value,
+            isNull(reference.product_category)
+                OR isNull(actual.product_category)
+                OR actual.physical_rows != 1
+                OR NOT isNotDistinctFrom(reference.orders, actual.orders)
+                OR NOT isNotDistinctFrom(reference.units, actual.units)
+                OR NOT isNotDistinctFrom(reference.revenue, actual.revenue)
+                OR NOT isNotDistinctFrom(reference.aov, actual.aov) AS failed
+        FROM reference
+        GLOBAL FULL OUTER JOIN actual USING (report_date, product_category)
     )
 SELECT
-    coalesce(reference.report_date, actual.report_date) AS data_date,
-    coalesce(reference.product_category, actual.product_category) AS business_key,
-    isNotNull(reference.product_category) AS reference_present,
-    isNotNull(actual.product_category) AS actual_present,
-    toString(tuple(reference.orders, reference.units, reference.revenue, reference.aov))
-        AS reference_value,
-    concat(
-        toString(tuple(actual.orders, actual.units, actual.revenue, actual.aov)),
-        ', rows=',
-        toString(actual.physical_rows)
-    ) AS actual_value,
-    isNull(reference.product_category)
-        OR isNull(actual.product_category)
-        OR actual.physical_rows != 1
-        OR NOT isNotDistinctFrom(reference.orders, actual.orders)
-        OR NOT isNotDistinctFrom(reference.units, actual.units)
-        OR NOT isNotDistinctFrom(reference.revenue, actual.revenue)
-        OR NOT isNotDistinctFrom(reference.aov, actual.aov) AS failed
-FROM reference
-GLOBAL FULL OUTER JOIN actual USING (report_date, product_category)
-ORDER BY
     data_date,
-    business_key
+    countIf(reference_present) AS reference_rows,
+    countIf(actual_present) AS actual_rows,
+    countIf(failed) AS failed_rows,
+    groupArrayIf(20)(
+        concat(
+            toString(data_date),
+            ' key=',
+            business_key,
+            ': эталон=',
+            if(reference_present, reference_value, 'нет ключа'),
+            ', объект=',
+            if(actual_present, actual_value, 'нет ключа')
+        ),
+        failed
+    ) AS diagnostics
+FROM comparison
+GROUP BY data_date
+ORDER BY data_date
 SETTINGS join_use_nulls = 1

sql/dq/sessions_vs_reference.sql

@@ -1,7 +1,7 @@
 -- DQ: эталонная сессия Метрики против сессии, нарезанной слоем DDS.
 --
--- Строка результата — одна кука одного завершённого дня. Сравнение по ключу
--- не даёт лишней сессии одной куки скрыть потерянную сессию другой.
+-- Сопоставление идет по куке внутри завершенного дня: лишняя сессия одной
+-- куки не скрывает потерянную сессию другой. Результат содержит итог дня.
 WITH
     (SELECT min(EventDate) FROM ods.event_v) AS first_source_day,
     first_source_day + ({position:UInt16} - 1) AS last_lived_day,
@@ -28,18 +28,38 @@ WITH
         GROUP BY
             data_date,
             client_id
+    ),
+    comparison AS
+    (
+        SELECT
+            coalesce(reference.data_date, actual.data_date) AS data_date,
+            toString(coalesce(reference.client_id, actual.client_id)) AS business_key,
+            isNotNull(reference.client_id) AS reference_present,
+            isNotNull(actual.client_id) AS actual_present,
+            toString(reference.sessions) AS reference_value,
+            toString(actual.sessions) AS actual_value,
+            NOT isNotDistinctFrom(reference.sessions, actual.sessions) AS failed
+        FROM reference
+        GLOBAL FULL OUTER JOIN actual USING (data_date, client_id)
     )
 SELECT
-    coalesce(reference.data_date, actual.data_date) AS data_date,
-    toString(coalesce(reference.client_id, actual.client_id)) AS business_key,
-    isNotNull(reference.client_id) AS reference_present,
-    isNotNull(actual.client_id) AS actual_present,
-    toString(reference.sessions) AS reference_value,
-    toString(actual.sessions) AS actual_value,
-    NOT isNotDistinctFrom(reference.sessions, actual.sessions) AS failed
-FROM reference
-GLOBAL FULL OUTER JOIN actual USING (data_date, client_id)
-ORDER BY
     data_date,
-    business_key
+    countIf(reference_present) AS reference_rows,
+    countIf(actual_present) AS actual_rows,
+    countIf(failed) AS failed_rows,
+    groupArrayIf(20)(
+        concat(
+            toString(data_date),
+            ' key=',
+            business_key,
+            ': эталон=',
+            if(reference_present, reference_value, 'нет ключа'),
+            ', объект=',
+            if(actual_present, actual_value, 'нет ключа')
+        ),
+        failed
+    ) AS diagnostics
+FROM comparison
+GROUP BY data_date
+ORDER BY data_date
 SETTINGS join_use_nulls = 1