fix(dq): устранено переполнение памяти проверок
- Зачем: - параллельный DQ выгружал деловые ключи в Python и исчерпывал 640 МиБ планировщика. - Что: - четыре сверки агрегируют дневные итоги и ограниченную диагностику в ClickHouse. - коробка планировщика поднята до 1,5 ГиБ, локальные пределы задач удалены. - измерения и граница между SQL и Python записаны в документации. - Проверка: - make lint, make config-test, make smoke, make check-clickhouse, make check-services. - полный etl_pipeline, красный и восстановительный прогоны dq_check.
This commit is contained in:
1 parent
26b2e5fea1
commit
54686651df
10 files changed
+295
-190
No files matched your search
@@ -57,39 +57,64 @@ WITH
|
||||
GROUP BY
|
||||
order_day,
|
||||
order_id
|
||||
),
|
||||
comparison AS
|
||||
(
|
||||
SELECT
|
||||
coalesce(reference.order_day, actual.order_day) AS data_date,
|
||||
coalesce(reference.order_id, actual.order_id) AS business_key,
|
||||
isNotNull(reference.order_id) AS reference_present,
|
||||
isNotNull(actual.order_id) AS actual_present,
|
||||
toString(tuple(
|
||||
reference.declared_revenue,
|
||||
reference.items_total,
|
||||
reference.status,
|
||||
reference.mismatch_class
|
||||
)) AS reference_value,
|
||||
concat(
|
||||
toString(tuple(
|
||||
actual.declared_revenue,
|
||||
actual.items_total,
|
||||
actual.status,
|
||||
actual.mismatch_class
|
||||
)),
|
||||
', rows=',
|
||||
toString(actual.physical_rows)
|
||||
) AS actual_value,
|
||||
isNull(reference.order_id)
|
||||
OR isNull(actual.order_id)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(
|
||||
reference.declared_revenue,
|
||||
actual.declared_revenue
|
||||
)
|
||||
OR NOT isNotDistinctFrom(reference.items_total, actual.items_total)
|
||||
OR NOT isNotDistinctFrom(reference.status, actual.status)
|
||||
OR NOT isNotDistinctFrom(
|
||||
reference.mismatch_class,
|
||||
actual.mismatch_class
|
||||
) AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (order_day, order_id)
|
||||
)
|
||||
SELECT
|
||||
coalesce(reference.order_day, actual.order_day) AS data_date,
|
||||
coalesce(reference.order_id, actual.order_id) AS business_key,
|
||||
isNotNull(reference.order_id) AS reference_present,
|
||||
isNotNull(actual.order_id) AS actual_present,
|
||||
toString(tuple(
|
||||
reference.declared_revenue,
|
||||
reference.items_total,
|
||||
reference.status,
|
||||
reference.mismatch_class
|
||||
)) AS reference_value,
|
||||
concat(
|
||||
toString(tuple(
|
||||
actual.declared_revenue,
|
||||
actual.items_total,
|
||||
actual.status,
|
||||
actual.mismatch_class
|
||||
)),
|
||||
', rows=',
|
||||
toString(actual.physical_rows)
|
||||
) AS actual_value,
|
||||
isNull(reference.order_id)
|
||||
OR isNull(actual.order_id)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.declared_revenue, actual.declared_revenue)
|
||||
OR NOT isNotDistinctFrom(reference.items_total, actual.items_total)
|
||||
OR NOT isNotDistinctFrom(reference.status, actual.status)
|
||||
OR NOT isNotDistinctFrom(reference.mismatch_class, actual.mismatch_class)
|
||||
AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (order_day, order_id)
|
||||
ORDER BY
|
||||
data_date,
|
||||
business_key
|
||||
countIf(reference_present) AS reference_rows,
|
||||
countIf(actual_present) AS actual_rows,
|
||||
countIf(failed) AS failed_rows,
|
||||
groupArrayIf(20)(
|
||||
concat(
|
||||
toString(data_date),
|
||||
' key=',
|
||||
business_key,
|
||||
': эталон=',
|
||||
if(reference_present, reference_value, 'нет ключа'),
|
||||
', объект=',
|
||||
if(actual_present, actual_value, 'нет ключа')
|
||||
),
|
||||
failed
|
||||
) AS diagnostics
|
||||
FROM comparison
|
||||
GROUP BY data_date
|
||||
ORDER BY data_date
|
||||
SETTINGS join_use_nulls = 1
|
||||
Reference in new issue
Block a user