fix(dq): устранено переполнение памяти проверок
- Зачем: - параллельный DQ выгружал деловые ключи в Python и исчерпывал 640 МиБ планировщика. - Что: - четыре сверки агрегируют дневные итоги и ограниченную диагностику в ClickHouse. - коробка планировщика поднята до 1,5 ГиБ, локальные пределы задач удалены. - измерения и граница между SQL и Python записаны в документации. - Проверка: - make lint, make config-test, make smoke, make check-clickhouse, make check-services. - полный etl_pipeline, красный и восстановительный прогоны dq_check.
This commit is contained in:
1 parent
26b2e5fea1
commit
54686651df
10 files changed
+295
-190
No files matched your search
@@ -23,24 +23,49 @@ WITH
|
||||
any(known_users) AS known_users
|
||||
FROM dm.daily_traffic_v
|
||||
GROUP BY report_date
|
||||
),
|
||||
comparison AS
|
||||
(
|
||||
SELECT
|
||||
coalesce(reference.report_date, actual.report_date) AS data_date,
|
||||
toString(coalesce(reference.report_date, actual.report_date))
|
||||
AS business_key,
|
||||
isNotNull(reference.report_date) AS reference_present,
|
||||
isNotNull(actual.report_date) AS actual_present,
|
||||
toString(tuple(reference.visitors, reference.known_users))
|
||||
AS reference_value,
|
||||
concat(
|
||||
toString(tuple(actual.visitors, actual.known_users)),
|
||||
', rows=',
|
||||
toString(actual.physical_rows)
|
||||
) AS actual_value,
|
||||
isNull(reference.report_date)
|
||||
OR isNull(actual.report_date)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.visitors, actual.visitors)
|
||||
OR NOT isNotDistinctFrom(reference.known_users, actual.known_users)
|
||||
AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (report_date)
|
||||
)
|
||||
SELECT
|
||||
coalesce(reference.report_date, actual.report_date) AS data_date,
|
||||
toString(coalesce(reference.report_date, actual.report_date)) AS business_key,
|
||||
isNotNull(reference.report_date) AS reference_present,
|
||||
isNotNull(actual.report_date) AS actual_present,
|
||||
toString(tuple(reference.visitors, reference.known_users)) AS reference_value,
|
||||
concat(
|
||||
toString(tuple(actual.visitors, actual.known_users)),
|
||||
', rows=',
|
||||
toString(actual.physical_rows)
|
||||
) AS actual_value,
|
||||
isNull(reference.report_date)
|
||||
OR isNull(actual.report_date)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.visitors, actual.visitors)
|
||||
OR NOT isNotDistinctFrom(reference.known_users, actual.known_users) AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (report_date)
|
||||
data_date,
|
||||
countIf(reference_present) AS reference_rows,
|
||||
countIf(actual_present) AS actual_rows,
|
||||
countIf(failed) AS failed_rows,
|
||||
groupArrayIf(20)(
|
||||
concat(
|
||||
toString(data_date),
|
||||
' key=',
|
||||
business_key,
|
||||
': эталон=',
|
||||
if(reference_present, reference_value, 'нет ключа'),
|
||||
', объект=',
|
||||
if(actual_present, actual_value, 'нет ключа')
|
||||
),
|
||||
failed
|
||||
) AS diagnostics
|
||||
FROM comparison
|
||||
GROUP BY data_date
|
||||
ORDER BY data_date
|
||||
SETTINGS join_use_nulls = 1
|
||||
Reference in new issue
Block a user