feat(dm): добавлены учебные задания витрин и даг самопроверки
- Зачем:
- менти строит витрины DM сам, от DDL по правилу зерна до задачи в
конвейере; стенду нужны ТЗ с образцом и обратная связь по каждой
витрине (#159, резолюция #135).
- Что:
- docs/labs/: README и пять ТЗ — events_enriched_v, top_pages_daily,
session_overview, dq_errors_daily, utm_effectiveness; каждое ТЗ
называет урок, состав колонок, форму по зерну и самопроверку.
- dags/homework_check.py: ручной даг, задача на витрину, красная без
витрины или при расхождении; проверки — шесть файлов sql/dq/ с
инвариантами против DDS и ODS, без золотых констант; в dq_summary
не пишет.
- docs: dm.md называет homework_check и снимает «Остается нарезке»,
§7 и §10 мастер-спеки согласованы с лабами, dds/session.md указывает
на ТЗ, AGENTS.md пополнен правилом имен docs/labs/.
- Проверка:
- make lint; make smoke 21/21, make check-clickhouse 10/10,
make check-services 7/7.
- на свежем стенде все пять задач homework_check красные и называют
ТЗ; черновик каждой витрины по тексту ТЗ зеленит свою задачу,
после сноса — снова красная; строк от самопроверки в dq_summary нет.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35d43125b1
commit
f33d75113d
17 files changed
+1367
-25
No files matched your search
@@ -0,0 +1,88 @@
|
||||
-- DQ: дневная сводка брака против живого окна таблиц ошибок ODS.
|
||||
--
|
||||
-- Граница окна повторяет TTL источников; более старую историю сверять не с чем.
|
||||
|
||||
WITH
|
||||
reference AS
|
||||
(
|
||||
SELECT
|
||||
load_date,
|
||||
source,
|
||||
error_class,
|
||||
count() AS error_rows
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
toDate(_load_ts) AS load_date,
|
||||
'event' AS source,
|
||||
error_class
|
||||
FROM ods.event_errors_dist
|
||||
UNION ALL
|
||||
SELECT
|
||||
toDate(_load_ts) AS load_date,
|
||||
'order' AS source,
|
||||
error_class
|
||||
FROM ods.order_errors_dist
|
||||
)
|
||||
WHERE load_date >= toDate(now() - INTERVAL 1 MONTH)
|
||||
GROUP BY load_date, source, error_class
|
||||
),
|
||||
actual AS
|
||||
(
|
||||
SELECT
|
||||
load_date,
|
||||
source,
|
||||
error_class,
|
||||
count() AS physical_rows,
|
||||
any(error_rows) AS error_rows
|
||||
FROM dm.dq_errors_daily_v
|
||||
WHERE load_date >= toDate(now() - INTERVAL 1 MONTH)
|
||||
GROUP BY load_date, source, error_class
|
||||
),
|
||||
comparison AS
|
||||
(
|
||||
SELECT
|
||||
coalesce(reference.load_date, actual.load_date) AS data_date,
|
||||
concat(
|
||||
coalesce(reference.source, actual.source),
|
||||
'/',
|
||||
coalesce(reference.error_class, actual.error_class)
|
||||
) AS business_key,
|
||||
isNotNull(reference.source) AS reference_present,
|
||||
isNotNull(actual.source) AS actual_present,
|
||||
reference.error_rows AS reference_value,
|
||||
actual.error_rows AS actual_value,
|
||||
actual.physical_rows AS physical_rows,
|
||||
isNull(reference.source)
|
||||
OR isNull(actual.source)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.error_rows, actual.error_rows)
|
||||
AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (load_date, source, error_class)
|
||||
)
|
||||
SELECT
|
||||
data_date,
|
||||
countIf(reference_present) AS reference_rows,
|
||||
countIf(actual_present) AS actual_rows,
|
||||
countIf(failed) AS failed_rows,
|
||||
groupArrayIf(20)(
|
||||
concat(
|
||||
toString(data_date),
|
||||
' source/error_class=',
|
||||
business_key,
|
||||
': источник=',
|
||||
if(reference_present, toString(reference_value), 'нет ключа'),
|
||||
', витрина=',
|
||||
if(
|
||||
actual_present,
|
||||
concat(toString(actual_value), ', rows=', toString(physical_rows)),
|
||||
'нет ключа'
|
||||
)
|
||||
),
|
||||
failed
|
||||
) AS diagnostics
|
||||
FROM comparison
|
||||
GROUP BY data_date
|
||||
ORDER BY data_date
|
||||
SETTINGS join_use_nulls = 1
|
||||
@@ -0,0 +1,15 @@
|
||||
-- DQ: область посуточной сверки обогащенных событий.
|
||||
--
|
||||
-- Объединение сторон оставляет видимым день, потерянный целиком.
|
||||
|
||||
SELECT day
|
||||
FROM
|
||||
(
|
||||
SELECT event_date AS day
|
||||
FROM dds.event_v
|
||||
UNION ALL
|
||||
SELECT event_date AS day
|
||||
FROM dm.events_enriched_v
|
||||
)
|
||||
GROUP BY day
|
||||
ORDER BY day
|
||||
@@ -0,0 +1,125 @@
|
||||
-- DQ: обогащенные события против прямого ответа DDS за один день.
|
||||
--
|
||||
-- День передает даг, чтобы широкая сверка не собирала всю историю в памяти.
|
||||
|
||||
WITH
|
||||
reference AS
|
||||
(
|
||||
SELECT
|
||||
event_date,
|
||||
event_id,
|
||||
tuple(
|
||||
event_time,
|
||||
toHour(event_time),
|
||||
client_id,
|
||||
event_type,
|
||||
url,
|
||||
path(url),
|
||||
referer,
|
||||
domain(referer),
|
||||
title,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
utm_content,
|
||||
utm_term,
|
||||
device_category,
|
||||
browser,
|
||||
operating_system,
|
||||
region_country,
|
||||
region_city,
|
||||
product_id,
|
||||
product_name,
|
||||
product_category,
|
||||
product_price,
|
||||
product_quantity,
|
||||
product_event_type,
|
||||
purchase_id,
|
||||
purchase_revenue,
|
||||
purchase_coupon
|
||||
) AS value
|
||||
FROM dds.event_v
|
||||
WHERE event_date = {day:Date}
|
||||
),
|
||||
actual AS
|
||||
(
|
||||
SELECT
|
||||
event_date,
|
||||
event_id,
|
||||
count() AS physical_rows,
|
||||
any(tuple(
|
||||
event_time,
|
||||
event_hour,
|
||||
client_id,
|
||||
event_type,
|
||||
url,
|
||||
page_path,
|
||||
referer,
|
||||
referer_host,
|
||||
title,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
utm_content,
|
||||
utm_term,
|
||||
device_category,
|
||||
browser,
|
||||
operating_system,
|
||||
region_country,
|
||||
region_city,
|
||||
product_id,
|
||||
product_name,
|
||||
product_category,
|
||||
product_price,
|
||||
product_quantity,
|
||||
product_event_type,
|
||||
purchase_id,
|
||||
purchase_revenue,
|
||||
purchase_coupon
|
||||
)) AS value
|
||||
FROM dm.events_enriched_v
|
||||
WHERE event_date = {day:Date}
|
||||
GROUP BY event_date, event_id
|
||||
),
|
||||
comparison AS
|
||||
(
|
||||
SELECT
|
||||
coalesce(reference.event_date, actual.event_date) AS data_date,
|
||||
coalesce(reference.event_id, actual.event_id) AS business_key,
|
||||
isNotNull(reference.event_id) AS reference_present,
|
||||
isNotNull(actual.event_id) AS actual_present,
|
||||
reference.value AS reference_value,
|
||||
actual.value AS actual_value,
|
||||
actual.physical_rows AS physical_rows,
|
||||
isNull(reference.event_id)
|
||||
OR isNull(actual.event_id)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.value, actual.value) AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (event_date, event_id)
|
||||
)
|
||||
SELECT
|
||||
data_date,
|
||||
countIf(reference_present) AS reference_rows,
|
||||
countIf(actual_present) AS actual_rows,
|
||||
countIf(failed) AS failed_rows,
|
||||
groupArrayIf(20)(
|
||||
concat(
|
||||
toString(data_date),
|
||||
' event_id=',
|
||||
toString(business_key),
|
||||
': источник=',
|
||||
if(reference_present, toString(reference_value), 'нет ключа'),
|
||||
', витрина=',
|
||||
if(
|
||||
actual_present,
|
||||
concat(toString(actual_value), ', rows=', toString(physical_rows)),
|
||||
'нет ключа'
|
||||
)
|
||||
),
|
||||
failed
|
||||
) AS diagnostics
|
||||
FROM comparison
|
||||
GROUP BY data_date
|
||||
ORDER BY data_date
|
||||
SETTINGS join_use_nulls = 1
|
||||
@@ -0,0 +1,87 @@
|
||||
-- DQ: дневной портрет сессий против прямого агрегата сессий DDS.
|
||||
--
|
||||
-- Полное объединение не прячет день или устройство, исчезнувшие с любой стороны.
|
||||
|
||||
WITH
|
||||
reference AS
|
||||
(
|
||||
SELECT
|
||||
session_date AS report_date,
|
||||
device_category,
|
||||
tuple(
|
||||
count(),
|
||||
uniqExact(client_id),
|
||||
countIf(events_total = 1),
|
||||
countIf(purchases > 0),
|
||||
sum(duration_seconds),
|
||||
sum(events_total),
|
||||
sum(pageviews),
|
||||
sum(cart_adds),
|
||||
sum(purchases)
|
||||
) AS value
|
||||
FROM dds.session_v
|
||||
GROUP BY report_date, device_category
|
||||
),
|
||||
actual AS
|
||||
(
|
||||
SELECT
|
||||
report_date,
|
||||
device_category,
|
||||
count() AS physical_rows,
|
||||
any(tuple(
|
||||
sessions,
|
||||
visitors,
|
||||
bounce_sessions,
|
||||
purchase_sessions,
|
||||
duration_seconds_total,
|
||||
events_total,
|
||||
pageviews,
|
||||
cart_adds,
|
||||
purchases
|
||||
)) AS value
|
||||
FROM dm.session_overview_v
|
||||
GROUP BY report_date, device_category
|
||||
),
|
||||
comparison AS
|
||||
(
|
||||
SELECT
|
||||
coalesce(reference.report_date, actual.report_date) AS data_date,
|
||||
coalesce(reference.device_category, actual.device_category)
|
||||
AS business_key,
|
||||
isNotNull(reference.device_category) AS reference_present,
|
||||
isNotNull(actual.device_category) AS actual_present,
|
||||
reference.value AS reference_value,
|
||||
actual.value AS actual_value,
|
||||
actual.physical_rows AS physical_rows,
|
||||
isNull(reference.device_category)
|
||||
OR isNull(actual.device_category)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.value, actual.value) AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (report_date, device_category)
|
||||
)
|
||||
SELECT
|
||||
data_date,
|
||||
countIf(reference_present) AS reference_rows,
|
||||
countIf(actual_present) AS actual_rows,
|
||||
countIf(failed) AS failed_rows,
|
||||
groupArrayIf(20)(
|
||||
concat(
|
||||
toString(data_date),
|
||||
' device_category=',
|
||||
business_key,
|
||||
': источник=',
|
||||
if(reference_present, toString(reference_value), 'нет ключа'),
|
||||
', витрина=',
|
||||
if(
|
||||
actual_present,
|
||||
concat(toString(actual_value), ', rows=', toString(physical_rows)),
|
||||
'нет ключа'
|
||||
)
|
||||
),
|
||||
failed
|
||||
) AS diagnostics
|
||||
FROM comparison
|
||||
GROUP BY data_date
|
||||
ORDER BY data_date
|
||||
SETTINGS join_use_nulls = 1
|
||||
@@ -0,0 +1,71 @@
|
||||
-- DQ: дневная популярность страниц против прямого агрегата событий DDS.
|
||||
--
|
||||
-- Полное объединение не прячет страницу, исчезнувшую с любой стороны.
|
||||
|
||||
WITH
|
||||
reference AS
|
||||
(
|
||||
SELECT
|
||||
event_date AS report_date,
|
||||
path(url) AS page_path,
|
||||
tuple(
|
||||
countIf(event_type = 'pageview'),
|
||||
uniqExact(client_id),
|
||||
countIf(event_type = 'add_to_cart'),
|
||||
countIf(event_type = 'purchase')
|
||||
) AS value
|
||||
FROM dds.event_v
|
||||
GROUP BY report_date, page_path
|
||||
),
|
||||
actual AS
|
||||
(
|
||||
SELECT
|
||||
report_date,
|
||||
page_path,
|
||||
count() AS physical_rows,
|
||||
any(tuple(pageviews, visitors, cart_adds, purchases)) AS value
|
||||
FROM dm.top_pages_daily_v
|
||||
GROUP BY report_date, page_path
|
||||
),
|
||||
comparison AS
|
||||
(
|
||||
SELECT
|
||||
coalesce(reference.report_date, actual.report_date) AS data_date,
|
||||
coalesce(reference.page_path, actual.page_path) AS business_key,
|
||||
isNotNull(reference.page_path) AS reference_present,
|
||||
isNotNull(actual.page_path) AS actual_present,
|
||||
reference.value AS reference_value,
|
||||
actual.value AS actual_value,
|
||||
actual.physical_rows AS physical_rows,
|
||||
isNull(reference.page_path)
|
||||
OR isNull(actual.page_path)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.value, actual.value) AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual USING (report_date, page_path)
|
||||
)
|
||||
SELECT
|
||||
data_date,
|
||||
countIf(reference_present) AS reference_rows,
|
||||
countIf(actual_present) AS actual_rows,
|
||||
countIf(failed) AS failed_rows,
|
||||
groupArrayIf(20)(
|
||||
concat(
|
||||
toString(data_date),
|
||||
' page_path=',
|
||||
business_key,
|
||||
': источник=',
|
||||
if(reference_present, toString(reference_value), 'нет ключа'),
|
||||
', витрина=',
|
||||
if(
|
||||
actual_present,
|
||||
concat(toString(actual_value), ', rows=', toString(physical_rows)),
|
||||
'нет ключа'
|
||||
)
|
||||
),
|
||||
failed
|
||||
) AS diagnostics
|
||||
FROM comparison
|
||||
GROUP BY data_date
|
||||
ORDER BY data_date
|
||||
SETTINGS join_use_nulls = 1
|
||||
@@ -0,0 +1,88 @@
|
||||
-- DQ: эффективность UTM-меток против прямого агрегата событий DDS.
|
||||
--
|
||||
-- Полное объединение не прячет набор меток, исчезнувший с любой стороны.
|
||||
|
||||
WITH
|
||||
reference AS
|
||||
(
|
||||
SELECT
|
||||
event_date AS report_date,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
tuple(
|
||||
uniqExact(client_id),
|
||||
countIf(event_type = 'pageview'),
|
||||
countIf(event_type = 'add_to_cart'),
|
||||
countIf(event_type = 'purchase'),
|
||||
sumIf(purchase_revenue[1], event_type = 'purchase')
|
||||
) AS value
|
||||
FROM dds.event_v
|
||||
GROUP BY report_date, utm_source, utm_medium, utm_campaign
|
||||
),
|
||||
actual AS
|
||||
(
|
||||
SELECT
|
||||
report_date,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
count() AS physical_rows,
|
||||
any(tuple(
|
||||
visitors,
|
||||
pageviews,
|
||||
add_to_carts,
|
||||
purchases,
|
||||
declared_revenue
|
||||
)) AS value
|
||||
FROM dm.utm_effectiveness_v
|
||||
GROUP BY report_date, utm_source, utm_medium, utm_campaign
|
||||
),
|
||||
comparison AS
|
||||
(
|
||||
SELECT
|
||||
coalesce(reference.report_date, actual.report_date) AS data_date,
|
||||
concat(
|
||||
'utm_source=', coalesce(reference.utm_source, actual.utm_source),
|
||||
', utm_medium=', coalesce(reference.utm_medium, actual.utm_medium),
|
||||
', utm_campaign=',
|
||||
coalesce(reference.utm_campaign, actual.utm_campaign)
|
||||
) AS business_key,
|
||||
isNotNull(reference.utm_source) AS reference_present,
|
||||
isNotNull(actual.utm_source) AS actual_present,
|
||||
reference.value AS reference_value,
|
||||
actual.value AS actual_value,
|
||||
actual.physical_rows AS physical_rows,
|
||||
isNull(reference.utm_source)
|
||||
OR isNull(actual.utm_source)
|
||||
OR actual.physical_rows != 1
|
||||
OR NOT isNotDistinctFrom(reference.value, actual.value) AS failed
|
||||
FROM reference
|
||||
GLOBAL FULL OUTER JOIN actual
|
||||
USING (report_date, utm_source, utm_medium, utm_campaign)
|
||||
)
|
||||
SELECT
|
||||
data_date,
|
||||
countIf(reference_present) AS reference_rows,
|
||||
countIf(actual_present) AS actual_rows,
|
||||
countIf(failed) AS failed_rows,
|
||||
groupArrayIf(20)(
|
||||
concat(
|
||||
toString(data_date),
|
||||
' ',
|
||||
business_key,
|
||||
': источник=',
|
||||
if(reference_present, toString(reference_value), 'нет ключа'),
|
||||
', витрина=',
|
||||
if(
|
||||
actual_present,
|
||||
concat(toString(actual_value), ', rows=', toString(physical_rows)),
|
||||
'нет ключа'
|
||||
)
|
||||
),
|
||||
failed
|
||||
) AS diagnostics
|
||||
FROM comparison
|
||||
GROUP BY data_date
|
||||
ORDER BY data_date
|
||||
SETTINGS join_use_nulls = 1
|
||||
Reference in new issue
Block a user