feat(infra): implement batch transformation layer and comprehensive documentation
Add batch ETL pipeline with ODS→DDS→DM transformation jobs and scripts. Create DDL infrastructure with automated database schema application. Update Makefile with transform target for executing batch processes. Rewrite README with complete Russian documentation including architecture diagrams, quick start guide, and data flow visualization.
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
-- Databases for layered DWH
|
||||
CREATE DATABASE IF NOT EXISTS stg;
|
||||
CREATE DATABASE IF NOT EXISTS ods;
|
||||
CREATE DATABASE IF NOT EXISTS dds;
|
||||
CREATE DATABASE IF NOT EXISTS dm;
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
-- STG layer: raw JSON storage + Kafka ingestion
|
||||
|
||||
-- Raw storage tables (target for MV from Kafka)
|
||||
CREATE TABLE IF NOT EXISTS stg.browser_raw
|
||||
(
|
||||
ingest_ts DateTime64(3) DEFAULT now64(3),
|
||||
kafka_topic LowCardinality(String) DEFAULT '',
|
||||
kafka_partition Int32 DEFAULT -1,
|
||||
kafka_offset Int64 DEFAULT -1,
|
||||
kafka_ts DateTime64(3) DEFAULT ingest_ts,
|
||||
raw String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (kafka_topic, kafka_partition, kafka_offset, ingest_ts);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stg.location_raw
|
||||
(
|
||||
ingest_ts DateTime64(3) DEFAULT now64(3),
|
||||
kafka_topic LowCardinality(String) DEFAULT '',
|
||||
kafka_partition Int32 DEFAULT -1,
|
||||
kafka_offset Int64 DEFAULT -1,
|
||||
kafka_ts DateTime64(3) DEFAULT ingest_ts,
|
||||
raw String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (kafka_topic, kafka_partition, kafka_offset, ingest_ts);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stg.device_raw
|
||||
(
|
||||
ingest_ts DateTime64(3) DEFAULT now64(3),
|
||||
kafka_topic LowCardinality(String) DEFAULT '',
|
||||
kafka_partition Int32 DEFAULT -1,
|
||||
kafka_offset Int64 DEFAULT -1,
|
||||
kafka_ts DateTime64(3) DEFAULT ingest_ts,
|
||||
raw String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (kafka_topic, kafka_partition, kafka_offset, ingest_ts);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stg.geo_raw
|
||||
(
|
||||
ingest_ts DateTime64(3) DEFAULT now64(3),
|
||||
kafka_topic LowCardinality(String) DEFAULT '',
|
||||
kafka_partition Int32 DEFAULT -1,
|
||||
kafka_offset Int64 DEFAULT -1,
|
||||
kafka_ts DateTime64(3) DEFAULT ingest_ts,
|
||||
raw String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (kafka_topic, kafka_partition, kafka_offset, ingest_ts);
|
||||
|
||||
-- Kafka source tables (ENGINE = Kafka)
|
||||
CREATE TABLE IF NOT EXISTS stg.kafka_browser_raw (raw String)
|
||||
ENGINE = Kafka
|
||||
SETTINGS
|
||||
kafka_broker_list = 'kafka:29092',
|
||||
kafka_topic_list = 'browser_events',
|
||||
kafka_group_name = 'ch_stg_browser',
|
||||
kafka_format = 'JSONAsString',
|
||||
kafka_num_consumers = 1,
|
||||
kafka_handle_error_mode = 'stream';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stg.kafka_location_raw (raw String)
|
||||
ENGINE = Kafka
|
||||
SETTINGS
|
||||
kafka_broker_list = 'kafka:29092',
|
||||
kafka_topic_list = 'location_events',
|
||||
kafka_group_name = 'ch_stg_location',
|
||||
kafka_format = 'JSONAsString',
|
||||
kafka_num_consumers = 1,
|
||||
kafka_handle_error_mode = 'stream';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stg.kafka_device_raw (raw String)
|
||||
ENGINE = Kafka
|
||||
SETTINGS
|
||||
kafka_broker_list = 'kafka:29092',
|
||||
kafka_topic_list = 'device_events',
|
||||
kafka_group_name = 'ch_stg_device',
|
||||
kafka_format = 'JSONAsString',
|
||||
kafka_num_consumers = 1,
|
||||
kafka_handle_error_mode = 'stream';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stg.kafka_geo_raw (raw String)
|
||||
ENGINE = Kafka
|
||||
SETTINGS
|
||||
kafka_broker_list = 'kafka:29092',
|
||||
kafka_topic_list = 'geo_events',
|
||||
kafka_group_name = 'ch_stg_geo',
|
||||
kafka_format = 'JSONAsString',
|
||||
kafka_num_consumers = 1,
|
||||
kafka_handle_error_mode = 'stream';
|
||||
|
||||
-- Materialized Views: Kafka → STG
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_kafka_browser_to_stg
|
||||
TO stg.browser_raw
|
||||
AS
|
||||
SELECT
|
||||
now64(3) AS ingest_ts,
|
||||
_topic AS kafka_topic,
|
||||
_partition AS kafka_partition,
|
||||
_offset AS kafka_offset,
|
||||
fromUnixTimestamp64Milli(toInt64(_timestamp_ms)) AS kafka_ts,
|
||||
raw
|
||||
FROM stg.kafka_browser_raw;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_kafka_location_to_stg
|
||||
TO stg.location_raw
|
||||
AS
|
||||
SELECT
|
||||
now64(3) AS ingest_ts,
|
||||
_topic AS kafka_topic,
|
||||
_partition AS kafka_partition,
|
||||
_offset AS kafka_offset,
|
||||
fromUnixTimestamp64Milli(toInt64(_timestamp_ms)) AS kafka_ts,
|
||||
raw
|
||||
FROM stg.kafka_location_raw;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_kafka_device_to_stg
|
||||
TO stg.device_raw
|
||||
AS
|
||||
SELECT
|
||||
now64(3) AS ingest_ts,
|
||||
_topic AS kafka_topic,
|
||||
_partition AS kafka_partition,
|
||||
_offset AS kafka_offset,
|
||||
fromUnixTimestamp64Milli(toInt64(_timestamp_ms)) AS kafka_ts,
|
||||
raw
|
||||
FROM stg.kafka_device_raw;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_kafka_geo_to_stg
|
||||
TO stg.geo_raw
|
||||
AS
|
||||
SELECT
|
||||
now64(3) AS ingest_ts,
|
||||
_topic AS kafka_topic,
|
||||
_partition AS kafka_partition,
|
||||
_offset AS kafka_offset,
|
||||
fromUnixTimestamp64Milli(toInt64(_timestamp_ms)) AS kafka_ts,
|
||||
raw
|
||||
FROM stg.kafka_geo_raw;
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
-- ODS layer: typed data + deduplication + DQ
|
||||
|
||||
-- Main ODS tables (valid keys only) + error tables (invalid keys)
|
||||
|
||||
-- ODS: browser_events
|
||||
CREATE TABLE IF NOT EXISTS ods.browser_event
|
||||
(
|
||||
event_id Nullable(UUID),
|
||||
event_ts Nullable(DateTime64(6)),
|
||||
event_date Date MATERIALIZED ifNull(toDate(event_ts), toDate(src_ingest_ts)),
|
||||
event_type LowCardinality(Nullable(String)),
|
||||
click_id Nullable(UUID),
|
||||
browser_name LowCardinality(Nullable(String)),
|
||||
browser_user_agent Nullable(String),
|
||||
browser_language LowCardinality(Nullable(String)),
|
||||
src_ingest_ts DateTime64(3),
|
||||
src_raw String,
|
||||
parse_errors Array(LowCardinality(String))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
||||
PARTITION BY toYYYYMM(event_date)
|
||||
ORDER BY (event_id)
|
||||
SETTINGS allow_nullable_key = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ods.browser_event_errors
|
||||
(
|
||||
ingest_ts DateTime64(3),
|
||||
kafka_topic LowCardinality(String),
|
||||
kafka_partition Int32,
|
||||
kafka_offset Int64,
|
||||
kafka_ts DateTime64(3),
|
||||
raw String,
|
||||
error_reason LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_browser_raw_to_ods
|
||||
TO ods.browser_event
|
||||
AS
|
||||
WITH
|
||||
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id,
|
||||
parseDateTime64BestEffortOrNull(JSONExtractString(raw, 'event_timestamp'), 6) AS event_ts,
|
||||
JSONExtractString(raw, 'event_type') AS event_type,
|
||||
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
||||
JSONExtractString(raw, 'browser_name') AS browser_name,
|
||||
JSONExtractString(raw, 'browser_user_agent') AS browser_user_agent,
|
||||
JSONExtractString(raw, 'browser_language') AS browser_language
|
||||
SELECT
|
||||
event_id,
|
||||
event_ts,
|
||||
event_type,
|
||||
click_id,
|
||||
browser_name,
|
||||
browser_user_agent,
|
||||
browser_language,
|
||||
ingest_ts AS src_ingest_ts,
|
||||
raw AS src_raw,
|
||||
arrayFilter(x -> x != '', [
|
||||
if(event_id IS NULL, 'bad_event_id', ''),
|
||||
if(event_ts IS NULL, 'bad_event_timestamp', ''),
|
||||
if(click_id IS NULL, 'bad_click_id', '')
|
||||
]) AS parse_errors
|
||||
FROM stg.browser_raw
|
||||
WHERE event_id IS NOT NULL; -- Filter NULL keys to error table
|
||||
|
||||
-- ODS: location_events
|
||||
CREATE TABLE IF NOT EXISTS ods.location_event
|
||||
(
|
||||
event_id Nullable(UUID),
|
||||
page_url Nullable(String),
|
||||
page_url_path LowCardinality(Nullable(String)),
|
||||
referer_url Nullable(String),
|
||||
referer_medium LowCardinality(Nullable(String)),
|
||||
utm_medium LowCardinality(Nullable(String)),
|
||||
utm_source LowCardinality(Nullable(String)),
|
||||
utm_content LowCardinality(Nullable(String)),
|
||||
utm_campaign LowCardinality(Nullable(String)),
|
||||
src_ingest_ts DateTime64(3),
|
||||
src_raw String,
|
||||
parse_errors Array(LowCardinality(String))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
||||
PARTITION BY toYYYYMM(toDate(src_ingest_ts))
|
||||
ORDER BY (event_id)
|
||||
SETTINGS allow_nullable_key = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ods.location_event_errors
|
||||
(
|
||||
ingest_ts DateTime64(3),
|
||||
kafka_topic LowCardinality(String),
|
||||
kafka_partition Int32,
|
||||
kafka_offset Int64,
|
||||
kafka_ts DateTime64(3),
|
||||
raw String,
|
||||
error_reason LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_location_raw_to_ods
|
||||
TO ods.location_event
|
||||
AS
|
||||
WITH
|
||||
toUUIDOrNull(JSONExtractString(raw, 'event_id')) AS event_id
|
||||
SELECT
|
||||
event_id,
|
||||
JSONExtractString(raw, 'page_url') AS page_url,
|
||||
JSONExtractString(raw, 'page_url_path') AS page_url_path,
|
||||
JSONExtractString(raw, 'referer_url') AS referer_url,
|
||||
JSONExtractString(raw, 'referer_medium') AS referer_medium,
|
||||
JSONExtractString(raw, 'utm_medium') AS utm_medium,
|
||||
JSONExtractString(raw, 'utm_source') AS utm_source,
|
||||
JSONExtractString(raw, 'utm_content') AS utm_content,
|
||||
JSONExtractString(raw, 'utm_campaign') AS utm_campaign,
|
||||
ingest_ts AS src_ingest_ts,
|
||||
raw AS src_raw,
|
||||
arrayFilter(x -> x != '', [
|
||||
if(event_id IS NULL, 'bad_event_id', '')
|
||||
]) AS parse_errors
|
||||
FROM stg.location_raw
|
||||
WHERE event_id IS NOT NULL;
|
||||
|
||||
-- ODS: device_events
|
||||
CREATE TABLE IF NOT EXISTS ods.device_by_click
|
||||
(
|
||||
click_id Nullable(UUID),
|
||||
os Nullable(String),
|
||||
os_name LowCardinality(Nullable(String)),
|
||||
os_timezone LowCardinality(Nullable(String)),
|
||||
device_type LowCardinality(Nullable(String)),
|
||||
device_is_mobile Nullable(UInt8),
|
||||
user_custom_id Nullable(String),
|
||||
user_domain_id Nullable(UUID),
|
||||
src_ingest_ts DateTime64(3),
|
||||
src_raw String,
|
||||
parse_errors Array(LowCardinality(String))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
||||
PARTITION BY toYYYYMM(toDate(src_ingest_ts))
|
||||
ORDER BY (click_id)
|
||||
SETTINGS allow_nullable_key = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ods.device_by_click_errors
|
||||
(
|
||||
ingest_ts DateTime64(3),
|
||||
kafka_topic LowCardinality(String),
|
||||
kafka_partition Int32,
|
||||
kafka_offset Int64,
|
||||
kafka_ts DateTime64(3),
|
||||
raw String,
|
||||
error_reason LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_device_raw_to_ods
|
||||
TO ods.device_by_click
|
||||
AS
|
||||
WITH
|
||||
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
||||
JSONExtract(raw, 'device_is_mobile', 'Nullable(UInt8)') AS device_is_mobile,
|
||||
toUUIDOrNull(JSONExtractString(raw, 'user_domain_id')) AS user_domain_id
|
||||
SELECT
|
||||
click_id,
|
||||
JSONExtractString(raw, 'os') AS os,
|
||||
JSONExtractString(raw, 'os_name') AS os_name,
|
||||
JSONExtractString(raw, 'os_timezone') AS os_timezone,
|
||||
JSONExtractString(raw, 'device_type') AS device_type,
|
||||
device_is_mobile,
|
||||
JSONExtractString(raw, 'user_custom_id') AS user_custom_id,
|
||||
user_domain_id,
|
||||
ingest_ts AS src_ingest_ts,
|
||||
raw AS src_raw,
|
||||
arrayFilter(x -> x != '', [
|
||||
if(click_id IS NULL, 'bad_click_id', ''),
|
||||
if(user_domain_id IS NULL, 'bad_user_domain_id', '')
|
||||
]) AS parse_errors
|
||||
FROM stg.device_raw
|
||||
WHERE click_id IS NOT NULL;
|
||||
|
||||
-- ODS: geo_events
|
||||
CREATE TABLE IF NOT EXISTS ods.geo_by_click
|
||||
(
|
||||
click_id Nullable(UUID),
|
||||
geo_latitude Nullable(Float64),
|
||||
geo_longitude Nullable(Float64),
|
||||
geo_country LowCardinality(Nullable(String)),
|
||||
geo_timezone LowCardinality(Nullable(String)),
|
||||
geo_region_name Nullable(String),
|
||||
ip_address Nullable(String),
|
||||
src_ingest_ts DateTime64(3),
|
||||
src_raw String,
|
||||
parse_errors Array(LowCardinality(String))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(src_ingest_ts)
|
||||
PARTITION BY toYYYYMM(toDate(src_ingest_ts))
|
||||
ORDER BY (click_id)
|
||||
SETTINGS allow_nullable_key = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ods.geo_by_click_errors
|
||||
(
|
||||
ingest_ts DateTime64(3),
|
||||
kafka_topic LowCardinality(String),
|
||||
kafka_partition Int32,
|
||||
kafka_offset Int64,
|
||||
kafka_ts DateTime64(3),
|
||||
raw String,
|
||||
error_reason LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(ingest_ts)
|
||||
ORDER BY (ingest_ts, kafka_topic, kafka_partition, kafka_offset);
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS stg.mv_geo_raw_to_ods
|
||||
TO ods.geo_by_click
|
||||
AS
|
||||
WITH
|
||||
toUUIDOrNull(JSONExtractString(raw, 'click_id')) AS click_id,
|
||||
toFloat64OrNull(JSONExtractString(raw, 'geo_latitude')) AS geo_latitude,
|
||||
toFloat64OrNull(JSONExtractString(raw, 'geo_longitude')) AS geo_longitude
|
||||
SELECT
|
||||
click_id,
|
||||
geo_latitude,
|
||||
geo_longitude,
|
||||
JSONExtractString(raw, 'geo_country') AS geo_country,
|
||||
JSONExtractString(raw, 'geo_timezone') AS geo_timezone,
|
||||
JSONExtractString(raw, 'geo_region_name') AS geo_region_name,
|
||||
JSONExtractString(raw, 'ip_address') AS ip_address,
|
||||
ingest_ts AS src_ingest_ts,
|
||||
raw AS src_raw,
|
||||
arrayFilter(x -> x != '', [
|
||||
if(click_id IS NULL, 'bad_click_id', ''),
|
||||
if(geo_latitude IS NULL, 'bad_geo_latitude', ''),
|
||||
if(geo_longitude IS NULL, 'bad_geo_longitude', '')
|
||||
]) AS parse_errors
|
||||
FROM stg.geo_raw
|
||||
WHERE click_id IS NOT NULL;
|
||||
@@ -0,0 +1,54 @@
|
||||
-- DDS layer: detailed entities (event + click context)
|
||||
-- Populated via batch SQL (not MV) to handle late arrivals and ensure consistency
|
||||
|
||||
-- DDS: click context (device + geo)
|
||||
CREATE TABLE IF NOT EXISTS dds.click
|
||||
(
|
||||
click_id UUID,
|
||||
user_domain_id Nullable(UUID),
|
||||
user_custom_id Nullable(String),
|
||||
device_type LowCardinality(Nullable(String)),
|
||||
device_is_mobile Nullable(UInt8),
|
||||
os_name LowCardinality(Nullable(String)),
|
||||
os Nullable(String),
|
||||
os_timezone LowCardinality(Nullable(String)),
|
||||
geo_country LowCardinality(Nullable(String)),
|
||||
geo_region_name Nullable(String),
|
||||
geo_timezone LowCardinality(Nullable(String)),
|
||||
geo_latitude Nullable(Float64),
|
||||
geo_longitude Nullable(Float64),
|
||||
ip_address Nullable(String),
|
||||
dds_update_ts DateTime64(3),
|
||||
ods_parse_errors Array(LowCardinality(String))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(dds_update_ts)
|
||||
PARTITION BY toYYYYMM(toDate(dds_update_ts))
|
||||
ORDER BY (click_id)
|
||||
SETTINGS allow_nullable_key = 1;
|
||||
|
||||
-- DDS: event (browser + location)
|
||||
CREATE TABLE IF NOT EXISTS dds.event
|
||||
(
|
||||
event_id UUID,
|
||||
event_ts Nullable(DateTime64(6)),
|
||||
event_date Date MATERIALIZED ifNull(toDate(event_ts), toDate(dds_update_ts)),
|
||||
event_type LowCardinality(Nullable(String)),
|
||||
click_id Nullable(UUID),
|
||||
page_url Nullable(String),
|
||||
page_url_path LowCardinality(Nullable(String)),
|
||||
referer_url Nullable(String),
|
||||
referer_medium LowCardinality(Nullable(String)),
|
||||
utm_medium LowCardinality(Nullable(String)),
|
||||
utm_source LowCardinality(Nullable(String)),
|
||||
utm_content LowCardinality(Nullable(String)),
|
||||
utm_campaign LowCardinality(Nullable(String)),
|
||||
browser_name LowCardinality(Nullable(String)),
|
||||
browser_user_agent Nullable(String),
|
||||
browser_language LowCardinality(Nullable(String)),
|
||||
dds_update_ts DateTime64(3),
|
||||
ods_parse_errors Array(LowCardinality(String))
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(dds_update_ts)
|
||||
PARTITION BY toYYYYMM(event_date)
|
||||
ORDER BY (event_id)
|
||||
SETTINGS allow_nullable_key = 1;
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
-- DM layer: Data Marts for BI (Superset)
|
||||
-- Views for enriched data and pre-computed aggregations
|
||||
|
||||
-- Main enriched view: event + click context
|
||||
CREATE VIEW IF NOT EXISTS dm.v_events_enriched AS
|
||||
SELECT
|
||||
e.event_id,
|
||||
e.event_ts,
|
||||
e.event_date,
|
||||
e.event_type,
|
||||
e.click_id,
|
||||
e.page_url,
|
||||
e.page_url_path,
|
||||
e.referer_url,
|
||||
e.referer_medium,
|
||||
e.utm_medium,
|
||||
e.utm_source,
|
||||
e.utm_content,
|
||||
e.utm_campaign,
|
||||
e.browser_name,
|
||||
e.browser_language,
|
||||
e.browser_user_agent,
|
||||
c.user_domain_id,
|
||||
c.user_custom_id,
|
||||
c.device_type,
|
||||
c.device_is_mobile,
|
||||
c.os_name,
|
||||
c.os_timezone,
|
||||
c.geo_country,
|
||||
c.geo_region_name,
|
||||
c.geo_timezone,
|
||||
c.geo_latitude,
|
||||
c.geo_longitude,
|
||||
c.ip_address,
|
||||
e.dds_update_ts,
|
||||
arrayConcat(e.ods_parse_errors, c.ods_parse_errors) AS parse_errors
|
||||
FROM dds.event AS e
|
||||
LEFT JOIN dds.click AS c ON c.click_id = e.click_id;
|
||||
|
||||
-- Daily traffic aggregation
|
||||
CREATE VIEW IF NOT EXISTS dm.v_daily_traffic AS
|
||||
SELECT
|
||||
event_date,
|
||||
geo_country,
|
||||
device_type,
|
||||
browser_name,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
count() AS events,
|
||||
uniqExact(click_id) AS uniq_clicks,
|
||||
uniqExact(user_domain_id) AS uniq_users
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_ts IS NOT NULL
|
||||
GROUP BY
|
||||
event_date,
|
||||
geo_country,
|
||||
device_type,
|
||||
browser_name,
|
||||
utm_source,
|
||||
utm_medium;
|
||||
|
||||
-- Top pages daily
|
||||
CREATE VIEW IF NOT EXISTS dm.v_top_pages_daily AS
|
||||
SELECT
|
||||
event_date,
|
||||
page_url_path,
|
||||
count() AS pageviews,
|
||||
uniqExact(click_id) AS uniq_clicks
|
||||
FROM dm.v_events_enriched
|
||||
WHERE event_type = 'pageview'
|
||||
GROUP BY event_date, page_url_path;
|
||||
|
||||
-- Data Quality errors daily
|
||||
CREATE VIEW IF NOT EXISTS dm.v_dq_errors_daily AS
|
||||
SELECT
|
||||
event_date,
|
||||
arrayJoin(parse_errors) AS error_code,
|
||||
count() AS rows_cnt
|
||||
FROM dm.v_events_enriched
|
||||
WHERE length(parse_errors) > 0
|
||||
GROUP BY event_date, error_code;
|
||||
|
||||
-- User sessions overview (approximate, by click_id within 30min windows)
|
||||
CREATE VIEW IF NOT EXISTS dm.v_session_overview AS
|
||||
SELECT
|
||||
event_date,
|
||||
user_domain_id,
|
||||
click_id,
|
||||
min(event_ts) AS session_start,
|
||||
max(event_ts) AS session_end,
|
||||
date_diff('minute', min(event_ts), max(event_ts)) AS session_duration_min,
|
||||
count() AS events_count,
|
||||
arrayDistinct(groupArray(page_url_path)) AS pages_visited,
|
||||
arrayDistinct(groupArray(geo_country)) AS countries,
|
||||
arrayDistinct(groupArray(device_type)) AS devices,
|
||||
groupArraySample(1, 1919)(utm_source)[1] AS utm_source_last,
|
||||
groupArraySample(1, 1919)(utm_medium)[1] AS utm_medium_last
|
||||
FROM dm.v_events_enriched
|
||||
WHERE user_domain_id IS NOT NULL
|
||||
GROUP BY event_date, user_domain_id, click_id;
|
||||
|
||||
-- UTM effectiveness (for marketing analysis)
|
||||
CREATE VIEW IF NOT EXISTS dm.v_utm_effectiveness AS
|
||||
SELECT
|
||||
event_date,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
count() AS clicks,
|
||||
uniqExact(user_domain_id) AS uniq_users,
|
||||
uniqExact(click_id) AS uniq_sessions,
|
||||
countIf(event_type = 'pageview') AS pageviews,
|
||||
countIf(event_type = 'purchase') AS purchases,
|
||||
countIf(event_type = 'add_to_cart') AS add_to_carts
|
||||
FROM dm.v_events_enriched
|
||||
WHERE utm_source IS NOT NULL OR utm_medium IS NOT NULL
|
||||
GROUP BY event_date, utm_source, utm_medium, utm_campaign;
|
||||
Reference in New Issue
Block a user