feat(infra): implement batch transformation layer and comprehensive documentation
Add batch ETL pipeline with ODS→DDS→DM transformation jobs and scripts. Create DDL infrastructure with automated database schema application. Update Makefile with transform target for executing batch processes. Rewrite README with complete Russian documentation including architecture diagrams, quick start guide, and data flow visualization.
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
-- Batch transformation: ODS → DDS
|
||||
-- Gets latest version from ODS (using argMax) and builds DDS entities
|
||||
-- Should be run periodically (e.g., every N minutes or via Airflow)
|
||||
|
||||
-- Refresh DDS.click (from device + geo)
|
||||
-- Strategy: full rebuild for demo; incremental for production
|
||||
INSERT INTO dds.click
|
||||
SELECT
|
||||
d.click_id,
|
||||
d.user_domain_id,
|
||||
d.user_custom_id,
|
||||
d.device_type,
|
||||
d.device_is_mobile,
|
||||
d.os_name,
|
||||
d.os,
|
||||
d.os_timezone,
|
||||
g.geo_country,
|
||||
g.geo_region_name,
|
||||
g.geo_timezone,
|
||||
g.geo_latitude,
|
||||
g.geo_longitude,
|
||||
g.ip_address,
|
||||
now64(3) AS dds_update_ts,
|
||||
arrayFilter(x -> x != '', arrayConcat(
|
||||
d.parse_errors,
|
||||
if(g.click_id IS NULL, ['geo_not_found'], []),
|
||||
if(g.geo_country IS NULL, ['geo_country_missing'], [])
|
||||
)) AS ods_parse_errors
|
||||
FROM (
|
||||
-- Latest device snapshot from ODS
|
||||
SELECT
|
||||
click_id,
|
||||
argMax(user_domain_id, src_ingest_ts) AS user_domain_id,
|
||||
argMax(user_custom_id, src_ingest_ts) AS user_custom_id,
|
||||
argMax(device_type, src_ingest_ts) AS device_type,
|
||||
argMax(device_is_mobile, src_ingest_ts) AS device_is_mobile,
|
||||
argMax(os_name, src_ingest_ts) AS os_name,
|
||||
argMax(os, src_ingest_ts) AS os,
|
||||
argMax(os_timezone, src_ingest_ts) AS os_timezone,
|
||||
argMax(parse_errors, src_ingest_ts) AS parse_errors
|
||||
FROM ods.device_by_click
|
||||
WHERE click_id IS NOT NULL
|
||||
GROUP BY click_id
|
||||
) AS d
|
||||
LEFT JOIN (
|
||||
-- Latest geo snapshot from ODS
|
||||
SELECT
|
||||
click_id,
|
||||
argMax(geo_country, src_ingest_ts) AS geo_country,
|
||||
argMax(geo_region_name, src_ingest_ts) AS geo_region_name,
|
||||
argMax(geo_timezone, src_ingest_ts) AS geo_timezone,
|
||||
argMax(geo_latitude, src_ingest_ts) AS geo_latitude,
|
||||
argMax(geo_longitude, src_ingest_ts) AS geo_longitude,
|
||||
argMax(ip_address, src_ingest_ts) AS ip_address
|
||||
FROM ods.geo_by_click
|
||||
WHERE click_id IS NOT NULL
|
||||
GROUP BY click_id
|
||||
) AS g ON g.click_id = d.click_id;
|
||||
|
||||
-- Refresh DDS.event (from browser + location)
|
||||
INSERT INTO dds.event
|
||||
SELECT
|
||||
b.event_id,
|
||||
b.event_ts,
|
||||
b.event_type,
|
||||
b.click_id,
|
||||
l.page_url,
|
||||
l.page_url_path,
|
||||
l.referer_url,
|
||||
l.referer_medium,
|
||||
l.utm_medium,
|
||||
l.utm_source,
|
||||
l.utm_content,
|
||||
l.utm_campaign,
|
||||
b.browser_name,
|
||||
b.browser_user_agent,
|
||||
b.browser_language,
|
||||
now64(3) AS dds_update_ts,
|
||||
arrayFilter(x -> x != '', arrayConcat(
|
||||
b.parse_errors,
|
||||
if(l.event_id IS NULL, ['location_not_found'], [])
|
||||
)) AS ods_parse_errors
|
||||
FROM (
|
||||
-- Latest browser snapshot from ODS
|
||||
SELECT
|
||||
event_id,
|
||||
argMax(event_ts, src_ingest_ts) AS event_ts,
|
||||
argMax(event_type, src_ingest_ts) AS event_type,
|
||||
argMax(click_id, src_ingest_ts) AS click_id,
|
||||
argMax(browser_name, src_ingest_ts) AS browser_name,
|
||||
argMax(browser_user_agent, src_ingest_ts) AS browser_user_agent,
|
||||
argMax(browser_language, src_ingest_ts) AS browser_language,
|
||||
argMax(parse_errors, src_ingest_ts) AS parse_errors
|
||||
FROM ods.browser_event
|
||||
WHERE event_id IS NOT NULL
|
||||
GROUP BY event_id
|
||||
) AS b
|
||||
LEFT JOIN (
|
||||
-- Latest location snapshot from ODS
|
||||
SELECT
|
||||
event_id,
|
||||
argMax(page_url, src_ingest_ts) AS page_url,
|
||||
argMax(page_url_path, src_ingest_ts) AS page_url_path,
|
||||
argMax(referer_url, src_ingest_ts) AS referer_url,
|
||||
argMax(referer_medium, src_ingest_ts) AS referer_medium,
|
||||
argMax(utm_medium, src_ingest_ts) AS utm_medium,
|
||||
argMax(utm_source, src_ingest_ts) AS utm_source,
|
||||
argMax(utm_content, src_ingest_ts) AS utm_content,
|
||||
argMax(utm_campaign, src_ingest_ts) AS utm_campaign
|
||||
FROM ods.location_event
|
||||
WHERE event_id IS NOT NULL
|
||||
GROUP BY event_id
|
||||
) AS l ON l.event_id = b.event_id;
|
||||
@@ -0,0 +1,72 @@
|
||||
-- Batch transformation: DDS → DM materialized tables
|
||||
-- For demo we use VIEWs mainly, but here we can materialize heavy aggregations
|
||||
|
||||
-- Materialized daily traffic (if needed for performance)
|
||||
-- Uncomment if VIEW dm.v_daily_traffic becomes too slow
|
||||
/*
|
||||
CREATE TABLE IF NOT EXISTS dm.daily_traffic_mart
|
||||
(
|
||||
event_date Date,
|
||||
geo_country LowCardinality(Nullable(String)),
|
||||
device_type LowCardinality(Nullable(String)),
|
||||
browser_name LowCardinality(Nullable(String)),
|
||||
utm_source LowCardinality(Nullable(String)),
|
||||
utm_medium LowCardinality(Nullable(String)),
|
||||
events UInt64,
|
||||
uniq_clicks UInt64,
|
||||
uniq_users UInt64
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(event_date)
|
||||
PARTITION BY toYYYYMM(event_date)
|
||||
ORDER BY (event_date, geo_country, device_type, browser_name, utm_source, utm_medium);
|
||||
|
||||
TRUNCATE TABLE dm.daily_traffic_mart;
|
||||
|
||||
INSERT INTO dm.daily_traffic_mart
|
||||
SELECT * FROM dm.v_daily_traffic;
|
||||
*/
|
||||
|
||||
-- Data Quality summary table (always fresh)
|
||||
CREATE TABLE IF NOT EXISTS dm.dq_summary
|
||||
(
|
||||
check_date Date,
|
||||
layer LowCardinality(String),
|
||||
table_name LowCardinality(String),
|
||||
check_name LowCardinality(String),
|
||||
check_value UInt64
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(check_date)
|
||||
ORDER BY (check_date, layer, table_name, check_name);
|
||||
|
||||
-- Truncate and refill DQ summary
|
||||
INSERT INTO dm.dq_summary
|
||||
SELECT
|
||||
today() AS check_date,
|
||||
'stg' AS layer,
|
||||
'browser_raw' AS table_name,
|
||||
'total_rows' AS check_name,
|
||||
count() AS check_value
|
||||
FROM stg.browser_raw
|
||||
UNION ALL
|
||||
SELECT today(), 'stg', 'location_raw', 'total_rows', count() FROM stg.location_raw
|
||||
UNION ALL
|
||||
SELECT today(), 'stg', 'device_raw', 'total_rows', count() FROM stg.device_raw
|
||||
UNION ALL
|
||||
SELECT today(), 'stg', 'geo_raw', 'total_rows', count() FROM stg.geo_raw
|
||||
UNION ALL
|
||||
SELECT today(), 'ods', 'browser_event', 'total_rows', count() FROM ods.browser_event
|
||||
UNION ALL
|
||||
SELECT today(), 'ods', 'browser_event', 'rows_with_errors', count() FROM ods.browser_event WHERE length(parse_errors) > 0
|
||||
UNION ALL
|
||||
SELECT today(), 'ods', 'location_event', 'total_rows', count() FROM ods.location_event
|
||||
UNION ALL
|
||||
SELECT today(), 'ods', 'device_by_click', 'total_rows', count() FROM ods.device_by_click
|
||||
UNION ALL
|
||||
SELECT today(), 'ods', 'geo_by_click', 'total_rows', count() FROM ods.geo_by_click
|
||||
UNION ALL
|
||||
SELECT today(), 'dds', 'event', 'total_rows', count() FROM dds.event
|
||||
UNION ALL
|
||||
SELECT today(), 'dds', 'click', 'total_rows', count() FROM dds.click
|
||||
UNION ALL
|
||||
SELECT today(), 'dds', 'event_without_click', 'orphan_events', count() FROM dds.event WHERE click_id IS NOT NULL AND click_id NOT IN (SELECT click_id FROM dds.click);
|
||||
Reference in New Issue
Block a user