ddadmin
e44b988d76
Add materialized views to capture parsing errors from browser, location, device, and geo raw staging tables and route them to dedicated error tables in the ODS layer. Refactor DDS refresh logic to handle partial data arrivals where device and geo events may arrive independently by using a unified click_id source with LEFT JOINs. Add TRUNCATE command to prevent duplicate data accumulation in DQ summary table.
343 lines
11 KiB
SQL
343 lines
11 KiB
SQL
-- ODS layer: typed data + deduplication + DQ
|
|
|
|
-- Main ODS tables (valid keys only) + error tables (invalid keys)
|
|
|
|
-- ODS: browser_events
|
|
CREATE TABLE IF NOT EXISTS ods.browser_event
|
|
(
|
|
event_id Nullable(UUID),
|
|
event_ts Nullable(DateTime64(6)),
|
|
event_date Date MATERIALIZED ifNull(toDate(event_ts), toDate(src_ingest_ts)),
|
|
event_type LowCardinality(Nullable(String)),
|
|
click_id Nullable(UUID),
|
|
browser_name LowCardinality(Nullable(String)),
|
|
browser_user_agent Nullable(String),
|
|
browser_language LowCardinality(Nullable(String)),
|
|
src_ingest_ts DateTime64(3),
|
|
src_raw String,
|
|
parse_errors Array(LowCardinality(String))
|
|
)
|
|
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
|
PARTITION BY toYYYYMM(event_date)
|
|
ORDER BY (event_id)
|
|
SETTINGS allow_nullable_key = 1;
|
|
|
|
CREATE TABLE IF NOT EXISTS ods.browser_event_errors
|
|
(
|
|
ingest_ts DateTime64(3),
|
|
kafka_topic LowCardinality(String),
|
|
kafka_partition Int32,
|
|
kafka_offset Int64,
|
|
kafka_ts DateTime64(3),
|
|
raw String,
|
|
error_reason LowCardinality(String)
|
|
)
|
|
ENGINE = MergeTree
|
|
PARTITION BY toYYYYMM(ingest_ts)
|
|
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_browser_raw_to_ods
|
|
TO ods.browser_event
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
|
|
parseDateTime64BestEffortOrNull(JSONExtractString(raw, 'event_timestamp'), 6) AS event_ts,
|
|
JSONExtractString(raw, 'event_type') AS event_type,
|
|
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
|
JSONExtractString(raw, 'browser_name') AS browser_name,
|
|
JSONExtractString(raw, 'browser_user_agent') AS browser_user_agent,
|
|
JSONExtractString(raw, 'browser_language') AS browser_language
|
|
SELECT
|
|
event_id,
|
|
event_ts,
|
|
event_type,
|
|
click_id,
|
|
browser_name,
|
|
browser_user_agent,
|
|
browser_language,
|
|
ingest_ts AS src_ingest_ts,
|
|
raw AS src_raw,
|
|
arrayFilter(x -> x != '', [
|
|
if(event_id IS NULL, 'bad_event_id', ''),
|
|
if(event_ts IS NULL, 'bad_event_timestamp', ''),
|
|
if(click_id IS NULL, 'bad_click_id', '')
|
|
]) AS parse_errors
|
|
FROM stg.browser_raw
|
|
WHERE event_id IS NOT NULL; -- Filter NULL keys to error table
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_browser_raw_to_ods_errors
|
|
TO ods.browser_event_errors
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
|
|
parseDateTime64BestEffortOrNull(JSONExtractString(raw, 'event_timestamp'), 6) AS event_ts,
|
|
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
|
arrayFilter(x -> x != '', [
|
|
if(event_id IS NULL, 'bad_event_id', ''),
|
|
if(event_ts IS NULL, 'bad_event_timestamp', ''),
|
|
if(click_id IS NULL, 'bad_click_id', '')
|
|
]) AS parse_errors
|
|
SELECT
|
|
ingest_ts,
|
|
kafka_topic,
|
|
kafka_partition,
|
|
kafka_offset,
|
|
kafka_ts,
|
|
raw,
|
|
arrayStringConcat(parse_errors, ',') AS error_reason
|
|
FROM stg.browser_raw
|
|
WHERE length(parse_errors) > 0
|
|
AND (
|
|
event_id IS NULL
|
|
OR event_ts IS NULL
|
|
OR click_id IS NULL
|
|
);
|
|
|
|
-- ODS: location_events
|
|
CREATE TABLE IF NOT EXISTS ods.location_event
|
|
(
|
|
event_id Nullable(UUID),
|
|
page_url Nullable(String),
|
|
page_url_path LowCardinality(Nullable(String)),
|
|
referer_url Nullable(String),
|
|
referer_medium LowCardinality(Nullable(String)),
|
|
utm_medium LowCardinality(Nullable(String)),
|
|
utm_source LowCardinality(Nullable(String)),
|
|
utm_content LowCardinality(Nullable(String)),
|
|
utm_campaign LowCardinality(Nullable(String)),
|
|
src_ingest_ts DateTime64(3),
|
|
src_raw String,
|
|
parse_errors Array(LowCardinality(String))
|
|
)
|
|
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
|
PARTITION BY toYYYYMM(toDate(src_ingest_ts))
|
|
ORDER BY (event_id)
|
|
SETTINGS allow_nullable_key = 1;
|
|
|
|
CREATE TABLE IF NOT EXISTS ods.location_event_errors
|
|
(
|
|
ingest_ts DateTime64(3),
|
|
kafka_topic LowCardinality(String),
|
|
kafka_partition Int32,
|
|
kafka_offset Int64,
|
|
kafka_ts DateTime64(3),
|
|
raw String,
|
|
error_reason LowCardinality(String)
|
|
)
|
|
ENGINE = MergeTree
|
|
PARTITION BY toYYYYMM(ingest_ts)
|
|
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_location_raw_to_ods
|
|
TO ods.location_event
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id
|
|
SELECT
|
|
event_id,
|
|
JSONExtractString(raw, 'page_url') AS page_url,
|
|
JSONExtractString(raw, 'page_url_path') AS page_url_path,
|
|
JSONExtractString(raw, 'referer_url') AS referer_url,
|
|
JSONExtractString(raw, 'referer_medium') AS referer_medium,
|
|
JSONExtractString(raw, 'utm_medium') AS utm_medium,
|
|
JSONExtractString(raw, 'utm_source') AS utm_source,
|
|
JSONExtractString(raw, 'utm_content') AS utm_content,
|
|
JSONExtractString(raw, 'utm_campaign') AS utm_campaign,
|
|
ingest_ts AS src_ingest_ts,
|
|
raw AS src_raw,
|
|
arrayFilter(x -> x != '', [
|
|
if(event_id IS NULL, 'bad_event_id', '')
|
|
]) AS parse_errors
|
|
FROM stg.location_raw
|
|
WHERE event_id IS NOT NULL;
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_location_raw_to_ods_errors
|
|
TO ods.location_event_errors
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
|
|
arrayFilter(x -> x != '', [
|
|
if(event_id IS NULL, 'bad_event_id', '')
|
|
]) AS parse_errors
|
|
SELECT
|
|
ingest_ts,
|
|
kafka_topic,
|
|
kafka_partition,
|
|
kafka_offset,
|
|
kafka_ts,
|
|
raw,
|
|
arrayStringConcat(parse_errors, ',') AS error_reason
|
|
FROM stg.location_raw
|
|
WHERE length(parse_errors) > 0
|
|
AND event_id IS NULL;
|
|
|
|
-- ODS: device_events
|
|
CREATE TABLE IF NOT EXISTS ods.device_by_click
|
|
(
|
|
click_id Nullable(UUID),
|
|
os Nullable(String),
|
|
os_name LowCardinality(Nullable(String)),
|
|
os_timezone LowCardinality(Nullable(String)),
|
|
device_type LowCardinality(Nullable(String)),
|
|
device_is_mobile Nullable(UInt8),
|
|
user_custom_id Nullable(String),
|
|
user_domain_id Nullable(UUID),
|
|
src_ingest_ts DateTime64(3),
|
|
src_raw String,
|
|
parse_errors Array(LowCardinality(String))
|
|
)
|
|
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
|
PARTITION BY toYYYYMM(toDate(src_ingest_ts))
|
|
ORDER BY (click_id)
|
|
SETTINGS allow_nullable_key = 1;
|
|
|
|
CREATE TABLE IF NOT EXISTS ods.device_by_click_errors
|
|
(
|
|
ingest_ts DateTime64(3),
|
|
kafka_topic LowCardinality(String),
|
|
kafka_partition Int32,
|
|
kafka_offset Int64,
|
|
kafka_ts DateTime64(3),
|
|
raw String,
|
|
error_reason LowCardinality(String)
|
|
)
|
|
ENGINE = MergeTree
|
|
PARTITION BY toYYYYMM(ingest_ts)
|
|
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_device_raw_to_ods
|
|
TO ods.device_by_click
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
|
JSONExtract(raw, 'device_is_mobile', 'Nullable(UInt8)') AS device_is_mobile,
|
|
toUUIDOrNull(JSONExtractString(raw, 'user_domain_id')) AS user_domain_id
|
|
SELECT
|
|
click_id,
|
|
JSONExtractString(raw, 'os') AS os,
|
|
JSONExtractString(raw, 'os_name') AS os_name,
|
|
JSONExtractString(raw, 'os_timezone') AS os_timezone,
|
|
JSONExtractString(raw, 'device_type') AS device_type,
|
|
device_is_mobile,
|
|
JSONExtractString(raw, 'user_custom_id') AS user_custom_id,
|
|
user_domain_id,
|
|
ingest_ts AS src_ingest_ts,
|
|
raw AS src_raw,
|
|
arrayFilter(x -> x != '', [
|
|
if(click_id IS NULL, 'bad_click_id', ''),
|
|
if(user_domain_id IS NULL, 'bad_user_domain_id', '')
|
|
]) AS parse_errors
|
|
FROM stg.device_raw
|
|
WHERE click_id IS NOT NULL;
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_device_raw_to_ods_errors
|
|
TO ods.device_by_click_errors
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
|
toUUIDOrNull(JSONExtractString(raw, 'user_domain_id')) AS user_domain_id,
|
|
arrayFilter(x -> x != '', [
|
|
if(click_id IS NULL, 'bad_click_id', ''),
|
|
if(user_domain_id IS NULL, 'bad_user_domain_id', '')
|
|
]) AS parse_errors
|
|
SELECT
|
|
ingest_ts,
|
|
kafka_topic,
|
|
kafka_partition,
|
|
kafka_offset,
|
|
kafka_ts,
|
|
raw,
|
|
arrayStringConcat(parse_errors, ',') AS error_reason
|
|
FROM stg.device_raw
|
|
WHERE length(parse_errors) > 0
|
|
AND (
|
|
click_id IS NULL
|
|
OR user_domain_id IS NULL
|
|
);
|
|
|
|
-- ODS: geo_events
|
|
CREATE TABLE IF NOT EXISTS ods.geo_by_click
|
|
(
|
|
click_id Nullable(UUID),
|
|
geo_latitude Nullable(Float64),
|
|
geo_longitude Nullable(Float64),
|
|
geo_country LowCardinality(Nullable(String)),
|
|
geo_timezone LowCardinality(Nullable(String)),
|
|
geo_region_name Nullable(String),
|
|
ip_address Nullable(String),
|
|
src_ingest_ts DateTime64(3),
|
|
src_raw String,
|
|
parse_errors Array(LowCardinality(String))
|
|
)
|
|
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
|
PARTITION BY toYYYYMM(toDate(src_ingest_ts))
|
|
ORDER BY (click_id)
|
|
SETTINGS allow_nullable_key = 1;
|
|
|
|
CREATE TABLE IF NOT EXISTS ods.geo_by_click_errors
|
|
(
|
|
ingest_ts DateTime64(3),
|
|
kafka_topic LowCardinality(String),
|
|
kafka_partition Int32,
|
|
kafka_offset Int64,
|
|
kafka_ts DateTime64(3),
|
|
raw String,
|
|
error_reason LowCardinality(String)
|
|
)
|
|
ENGINE = MergeTree
|
|
PARTITION BY toYYYYMM(ingest_ts)
|
|
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_geo_raw_to_ods
|
|
TO ods.geo_by_click
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
|
toFloat64OrNull(JSONExtractString(raw, 'geo_latitude')) AS geo_latitude,
|
|
toFloat64OrNull(JSONExtractString(raw, 'geo_longitude')) AS geo_longitude
|
|
SELECT
|
|
click_id,
|
|
geo_latitude,
|
|
geo_longitude,
|
|
JSONExtractString(raw, 'geo_country') AS geo_country,
|
|
JSONExtractString(raw, 'geo_timezone') AS geo_timezone,
|
|
JSONExtractString(raw, 'geo_region_name') AS geo_region_name,
|
|
JSONExtractString(raw, 'ip_address') AS ip_address,
|
|
ingest_ts AS src_ingest_ts,
|
|
raw AS src_raw,
|
|
arrayFilter(x -> x != '', [
|
|
if(click_id IS NULL, 'bad_click_id', ''),
|
|
if(geo_latitude IS NULL, 'bad_geo_latitude', ''),
|
|
if(geo_longitude IS NULL, 'bad_geo_longitude', '')
|
|
]) AS parse_errors
|
|
FROM stg.geo_raw
|
|
WHERE click_id IS NOT NULL;
|
|
|
|
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_geo_raw_to_ods_errors
|
|
TO ods.geo_by_click_errors
|
|
AS
|
|
WITH
|
|
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
|
toFloat64OrNull(JSONExtractString(raw, 'geo_latitude')) AS geo_latitude,
|
|
toFloat64OrNull(JSONExtractString(raw, 'geo_longitude')) AS geo_longitude,
|
|
arrayFilter(x -> x != '', [
|
|
if(click_id IS NULL, 'bad_click_id', ''),
|
|
if(geo_latitude IS NULL, 'bad_geo_latitude', ''),
|
|
if(geo_longitude IS NULL, 'bad_geo_longitude', '')
|
|
]) AS parse_errors
|
|
SELECT
|
|
ingest_ts,
|
|
kafka_topic,
|
|
kafka_partition,
|
|
kafka_offset,
|
|
kafka_ts,
|
|
raw,
|
|
arrayStringConcat(parse_errors, ',') AS error_reason
|
|
FROM stg.geo_raw
|
|
WHERE length(parse_errors) > 0
|
|
AND (
|
|
click_id IS NULL
|
|
OR geo_latitude IS NULL
|
|
OR geo_longitude IS NULL
|
|
);
|