ddmitry/airflow-greenplum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ods): реализован ODS слой и DAG загрузки из STG

Browse Source 
- Зачем:
  - подготовлена учебная реализация ODS слоя с типизацией, UPSERT и DQ, чтобы продолжить работу от STG к DDS/DM.
- Что:
  - добавлены SQL-скрипты `sql/ods/*_ddl.sql`, `sql/ods/*_load.sql`, `sql/ods/*_dq.sql` для 9 сущностей bookings.
  - добавлены DAG `bookings_ods_ddl` и `bookings_to_gp_ods`, а также smoke-тесты для новых графов.
  - ODS DDL интегрирован в `sql/ddl_gp.sql`; документация и план обновлены под единый запуск через `make ddl-gp`.
- Проверка:
  - `make test`.
  - `make ddl-gp`.
This commit is contained in:
ddadmin
2026-02-28 22:13:11 +03:00
parent 2d629b6e80
commit 85fea19818
37 changed files with 2704 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sql/ods/airplanes_ddl.sql
+13
View File
@@ -0,0 +1,13 @@
-- DDL для ODS-слоя по таблице airplanes (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.airplanes (
airplane_code TEXT NOT NULL,
model TEXT NOT NULL,
range_km INTEGER,
speed_kmh INTEGER,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (airplane_code);
sql/ods/airplanes_dq.sql
+99
View File
@@ -0,0 +1,99 @@
-- DQ для ODS airplanes.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_extra_keys_count BIGINT;
v_null_count BIGINT;
BEGIN
-- Для snapshot-справочников пустой батч — ошибка.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.airplanes
WHERE batch_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.airplanes пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT airplane_code)
INTO v_dup_count
FROM ods.airplanes;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airplanes найдены дубликаты airplane_code: %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT airplane_code
FROM stg.airplanes
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.airplanes AS o
WHERE o.airplane_code = s.airplane_code
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airplanes отсутствуют ключи из stg.airplanes (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- В ODS не должно быть лишних ключей, которых нет в snapshot текущего батча.
SELECT COUNT(*)
INTO v_extra_keys_count
FROM ods.airplanes AS o
WHERE NOT EXISTS (
SELECT 1
FROM (
SELECT DISTINCT airplane_code
FROM stg.airplanes
WHERE batch_id = v_batch_id
) AS s
WHERE s.airplane_code = o.airplane_code
);
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airplanes найдены лишние ключи вне stg batch_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.airplanes
WHERE airplane_code IS NULL
OR airplane_code = ''
OR model IS NULL
OR model = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airplanes найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.airplanes ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/airplanes_load.sql
+92
View File
@@ -0,0 +1,92 @@
-- Загрузка ODS по airplanes: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.airplane_code,
s.model,
NULLIF(s.range, '')::INTEGER AS range_km,
NULLIF(s.speed, '')::INTEGER AS speed_kmh,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.airplanes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.airplanes AS o
SET model = s.model,
range_km = s.range_km,
speed_kmh = s.speed_kmh,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.airplane_code = s.airplane_code
AND (
o.model IS DISTINCT FROM s.model
OR o.range_km IS DISTINCT FROM s.range_km
OR o.speed_kmh IS DISTINCT FROM s.speed_kmh
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.airplane_code,
s.model,
NULLIF(s.range, '')::INTEGER AS range_km,
NULLIF(s.speed, '')::INTEGER AS speed_kmh,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.airplanes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.airplanes (
airplane_code,
model,
range_km,
speed_kmh,
_load_id,
_load_ts
)
SELECT
s.airplane_code,
s.model,
s.range_km,
s.speed_kmh,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.airplanes AS o
WHERE o.airplane_code = s.airplane_code
);
-- Statement 3: DELETE ключей, которых нет в snapshot текущего батча.
-- Это делает ODS для справочника действительно "current state".
WITH src_keys AS (
SELECT d.airplane_code
FROM (
SELECT
s.airplane_code,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.airplanes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
) AS d
WHERE d.rn = 1
)
DELETE FROM ods.airplanes AS o
WHERE NOT EXISTS (
SELECT 1
FROM src_keys AS s
WHERE s.airplane_code = o.airplane_code
);
ANALYZE ods.airplanes;
sql/ods/airports_ddl.sql
+15
View File
@@ -0,0 +1,15 @@
-- DDL для ODS-слоя по таблице airports (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.airports (
airport_code TEXT NOT NULL,
airport_name TEXT NOT NULL,
city TEXT NOT NULL,
country TEXT NOT NULL,
coordinates TEXT,
timezone TEXT NOT NULL,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (airport_code);
sql/ods/airports_dq.sql
+105
View File
@@ -0,0 +1,105 @@
-- DQ для ODS airports.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_extra_keys_count BIGINT;
v_null_count BIGINT;
BEGIN
-- Для snapshot-справочников пустой батч — ошибка.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.airports
WHERE batch_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.airports пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT airport_code)
INTO v_dup_count
FROM ods.airports;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airports найдены дубликаты airport_code: %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT airport_code
FROM stg.airports
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.airports AS o
WHERE o.airport_code = s.airport_code
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airports отсутствуют ключи из stg.airports (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- В ODS не должно быть лишних ключей, которых нет в snapshot текущего батча.
SELECT COUNT(*)
INTO v_extra_keys_count
FROM ods.airports AS o
WHERE NOT EXISTS (
SELECT 1
FROM (
SELECT DISTINCT airport_code
FROM stg.airports
WHERE batch_id = v_batch_id
) AS s
WHERE s.airport_code = o.airport_code
);
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airports найдены лишние ключи вне stg batch_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.airports
WHERE airport_code IS NULL
OR airport_code = ''
OR airport_name IS NULL
OR airport_name = ''
OR city IS NULL
OR city = ''
OR country IS NULL
OR country = ''
OR timezone IS NULL
OR timezone = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airports найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.airports ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/airports_load.sql
+104
View File
@@ -0,0 +1,104 @@
-- Загрузка ODS по airports: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.airport_code,
s.airport_name,
s.city,
s.country,
s.coordinates,
s.timezone,
ROW_NUMBER() OVER (
PARTITION BY s.airport_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.airports AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.airports AS o
SET airport_name = s.airport_name,
city = s.city,
country = s.country,
coordinates = s.coordinates,
timezone = s.timezone,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.airport_code = s.airport_code
AND (
o.airport_name IS DISTINCT FROM s.airport_name
OR o.city IS DISTINCT FROM s.city
OR o.country IS DISTINCT FROM s.country
OR o.coordinates IS DISTINCT FROM s.coordinates
OR o.timezone IS DISTINCT FROM s.timezone
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.airport_code,
s.airport_name,
s.city,
s.country,
s.coordinates,
s.timezone,
ROW_NUMBER() OVER (
PARTITION BY s.airport_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.airports AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.airports (
airport_code,
airport_name,
city,
country,
coordinates,
timezone,
_load_id,
_load_ts
)
SELECT
s.airport_code,
s.airport_name,
s.city,
s.country,
s.coordinates,
s.timezone,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.airports AS o
WHERE o.airport_code = s.airport_code
);
-- Statement 3: DELETE ключей, которых нет в snapshot текущего батча.
-- Это делает ODS для справочника действительно "current state".
WITH src_keys AS (
SELECT d.airport_code
FROM (
SELECT
s.airport_code,
ROW_NUMBER() OVER (
PARTITION BY s.airport_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.airports AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
) AS d
WHERE d.rn = 1
)
DELETE FROM ods.airports AS o
WHERE NOT EXISTS (
SELECT 1
FROM src_keys AS s
WHERE s.airport_code = o.airport_code
);
ANALYZE ods.airports;
sql/ods/boarding_passes_ddl.sql
+15
View File
@@ -0,0 +1,15 @@
-- DDL для ODS-слоя по таблице boarding_passes (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.boarding_passes (
ticket_no TEXT NOT NULL,
flight_id INTEGER NOT NULL,
seat_no TEXT NOT NULL,
boarding_no INTEGER,
boarding_time TIMESTAMP WITH TIME ZONE,
event_ts TIMESTAMP,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (ticket_no);
sql/ods/boarding_passes_dq.sql
+96
View File
@@ -0,0 +1,96 @@
-- DQ для ODS boarding_passes.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_null_count BIGINT;
v_orphan_segment_count BIGINT;
BEGIN
-- Для этой таблицы пустой батч допустим.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.boarding_passes
WHERE batch_id = v_batch_id;
-- В ODS не должно быть дублей по составному бизнес-ключу.
SELECT COUNT(*)
INTO v_dup_count
FROM (
SELECT ticket_no, flight_id
FROM ods.boarding_passes
GROUP BY ticket_no, flight_id
HAVING COUNT(*) > 1
) AS d;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.boarding_passes найдены дубликаты (ticket_no, flight_id): %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT ticket_no, NULLIF(flight_id, '')::INTEGER AS flight_id
FROM stg.boarding_passes
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.boarding_passes AS o
WHERE o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.boarding_passes отсутствуют ключи из stg.boarding_passes (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.boarding_passes
WHERE ticket_no IS NULL
OR ticket_no = ''
OR flight_id IS NULL
OR seat_no IS NULL
OR seat_no = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.boarding_passes найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
-- Ссылочная целостность: boarding_passes (ticket_no, flight_id) -> segments.
SELECT COUNT(*)
INTO v_orphan_segment_count
FROM ods.boarding_passes AS bp
WHERE NOT EXISTS (
SELECT 1
FROM ods.segments AS s
WHERE s.ticket_no = bp.ticket_no
AND s.flight_id = bp.flight_id
);
IF v_orphan_segment_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.boarding_passes найдены строки без соответствующего ods.segments: %',
v_orphan_segment_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.boarding_passes ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/boarding_passes_load.sql
+81
View File
@@ -0,0 +1,81 @@
-- Загрузка ODS по boarding_passes: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.ticket_no,
NULLIF(s.flight_id, '')::INTEGER AS flight_id,
s.seat_no,
NULLIF(s.boarding_no, '')::INTEGER AS boarding_no,
NULLIF(s.boarding_time, '')::TIMESTAMP WITH TIME ZONE AS boarding_time,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no, s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.boarding_passes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.boarding_passes AS o
SET seat_no = s.seat_no,
boarding_no = s.boarding_no,
boarding_time = s.boarding_time,
event_ts = s.event_ts,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
AND (
o.seat_no IS DISTINCT FROM s.seat_no
OR o.boarding_no IS DISTINCT FROM s.boarding_no
OR o.boarding_time IS DISTINCT FROM s.boarding_time
OR o.event_ts IS DISTINCT FROM s.event_ts
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.ticket_no,
NULLIF(s.flight_id, '')::INTEGER AS flight_id,
s.seat_no,
NULLIF(s.boarding_no, '')::INTEGER AS boarding_no,
NULLIF(s.boarding_time, '')::TIMESTAMP WITH TIME ZONE AS boarding_time,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no, s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.boarding_passes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.boarding_passes (
ticket_no,
flight_id,
seat_no,
boarding_no,
boarding_time,
event_ts,
_load_id,
_load_ts
)
SELECT
s.ticket_no,
s.flight_id,
s.seat_no,
s.boarding_no,
s.boarding_time,
s.event_ts,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.boarding_passes AS o
WHERE o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
);
ANALYZE ods.boarding_passes;
sql/ods/bookings_ddl.sql
+13
View File
@@ -0,0 +1,13 @@
-- DDL для ODS-слоя по таблице bookings (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.bookings (
book_ref TEXT NOT NULL,
book_date TIMESTAMP WITH TIME ZONE NOT NULL,
total_amount NUMERIC(10,2) NOT NULL,
event_ts TIMESTAMP,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (book_ref);
sql/ods/bookings_dq.sql
+71
View File
@@ -0,0 +1,71 @@
-- DQ для ODS bookings.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_null_count BIGINT;
BEGIN
-- Для инкрементальных таблиц пустой батч допустим.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.bookings
WHERE batch_id = v_batch_id;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT book_ref)
INTO v_dup_count
FROM ods.bookings;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.bookings найдены дубликаты book_ref: %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT book_ref
FROM stg.bookings
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.bookings AS o
WHERE o.book_ref = s.book_ref
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.bookings отсутствуют ключи из stg.bookings (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.bookings
WHERE book_ref IS NULL
OR book_ref = ''
OR book_date IS NULL
OR total_amount IS NULL
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.bookings найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.bookings ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/bookings_load.sql
+69
View File
@@ -0,0 +1,69 @@
-- Загрузка ODS по bookings: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.book_ref,
NULLIF(s.book_date, '')::TIMESTAMP WITH TIME ZONE AS book_date,
NULLIF(s.total_amount, '')::NUMERIC(10,2) AS total_amount,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.book_ref
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.bookings AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.bookings AS o
SET book_date = s.book_date,
total_amount = s.total_amount,
event_ts = s.event_ts,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.book_ref = s.book_ref
AND (
o.book_date IS DISTINCT FROM s.book_date
OR o.total_amount IS DISTINCT FROM s.total_amount
OR o.event_ts IS DISTINCT FROM s.event_ts
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.book_ref,
NULLIF(s.book_date, '')::TIMESTAMP WITH TIME ZONE AS book_date,
NULLIF(s.total_amount, '')::NUMERIC(10,2) AS total_amount,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.book_ref
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.bookings AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.bookings (
book_ref,
book_date,
total_amount,
event_ts,
_load_id,
_load_ts
)
SELECT
s.book_ref,
s.book_date,
s.total_amount,
s.event_ts,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.bookings AS o
WHERE o.book_ref = s.book_ref
);
ANALYZE ods.bookings;
sql/ods/flights_ddl.sql
+17
View File
@@ -0,0 +1,17 @@
-- DDL для ODS-слоя по таблице flights (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.flights (
flight_id INTEGER NOT NULL,
route_no TEXT NOT NULL,
status TEXT NOT NULL,
scheduled_departure TIMESTAMP WITH TIME ZONE,
scheduled_arrival TIMESTAMP WITH TIME ZONE,
actual_departure TIMESTAMP WITH TIME ZONE,
actual_arrival TIMESTAMP WITH TIME ZONE,
event_ts TIMESTAMP,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (flight_id);
sql/ods/flights_dq.sql
+89
View File
@@ -0,0 +1,89 @@
-- DQ для ODS flights.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_null_count BIGINT;
v_orphan_route_count BIGINT;
BEGIN
-- Для инкрементальных таблиц пустой батч допустим.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.flights
WHERE batch_id = v_batch_id;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT flight_id)
INTO v_dup_count
FROM ods.flights;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.flights найдены дубликаты flight_id: %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT NULLIF(flight_id, '')::INTEGER AS flight_id
FROM stg.flights
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.flights AS o
WHERE o.flight_id = s.flight_id
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.flights отсутствуют ключи из stg.flights (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.flights
WHERE flight_id IS NULL
OR route_no IS NULL
OR route_no = ''
OR status IS NULL
OR status = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.flights найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
-- Ссылочная целостность: flights.route_no -> routes.route_no (упрощённо, без validity).
SELECT COUNT(*)
INTO v_orphan_route_count
FROM ods.flights AS f
WHERE NOT EXISTS (
SELECT 1
FROM ods.routes AS r
WHERE r.route_no = f.route_no
);
IF v_orphan_route_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.flights найдены строки без соответствующего routes.route_no: %',
v_orphan_route_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.flights ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/flights_load.sql
+93
View File
@@ -0,0 +1,93 @@
-- Загрузка ODS по flights: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
NULLIF(s.flight_id, '')::INTEGER AS flight_id,
s.route_no,
s.status,
NULLIF(s.scheduled_departure, '')::TIMESTAMP WITH TIME ZONE AS scheduled_departure,
NULLIF(s.scheduled_arrival, '')::TIMESTAMP WITH TIME ZONE AS scheduled_arrival,
NULLIF(s.actual_departure, '')::TIMESTAMP WITH TIME ZONE AS actual_departure,
NULLIF(s.actual_arrival, '')::TIMESTAMP WITH TIME ZONE AS actual_arrival,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.flights AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.flights AS o
SET route_no = s.route_no,
status = s.status,
scheduled_departure = s.scheduled_departure,
scheduled_arrival = s.scheduled_arrival,
actual_departure = s.actual_departure,
actual_arrival = s.actual_arrival,
event_ts = s.event_ts,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.flight_id = s.flight_id
AND (
o.route_no IS DISTINCT FROM s.route_no
OR o.status IS DISTINCT FROM s.status
OR o.scheduled_departure IS DISTINCT FROM s.scheduled_departure
OR o.scheduled_arrival IS DISTINCT FROM s.scheduled_arrival
OR o.actual_departure IS DISTINCT FROM s.actual_departure
OR o.actual_arrival IS DISTINCT FROM s.actual_arrival
OR o.event_ts IS DISTINCT FROM s.event_ts
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
NULLIF(s.flight_id, '')::INTEGER AS flight_id,
s.route_no,
s.status,
NULLIF(s.scheduled_departure, '')::TIMESTAMP WITH TIME ZONE AS scheduled_departure,
NULLIF(s.scheduled_arrival, '')::TIMESTAMP WITH TIME ZONE AS scheduled_arrival,
NULLIF(s.actual_departure, '')::TIMESTAMP WITH TIME ZONE AS actual_departure,
NULLIF(s.actual_arrival, '')::TIMESTAMP WITH TIME ZONE AS actual_arrival,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.flights AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.flights (
flight_id,
route_no,
status,
scheduled_departure,
scheduled_arrival,
actual_departure,
actual_arrival,
event_ts,
_load_id,
_load_ts
)
SELECT
s.flight_id,
s.route_no,
s.status,
s.scheduled_departure,
s.scheduled_arrival,
s.actual_departure,
s.actual_arrival,
s.event_ts,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.flights AS o
WHERE o.flight_id = s.flight_id
);
ANALYZE ods.flights;
sql/ods/routes_ddl.sql
+17
View File
@@ -0,0 +1,17 @@
-- DDL для ODS-слоя по таблице routes (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.routes (
route_no TEXT NOT NULL,
validity TEXT NOT NULL,
departure_airport TEXT NOT NULL,
arrival_airport TEXT NOT NULL,
airplane_code TEXT NOT NULL,
days_of_week TEXT,
departure_time TIME,
duration INTERVAL,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (route_no);
sql/ods/routes_dq.sql
+163
View File
@@ -0,0 +1,163 @@
-- DQ для ODS routes.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_extra_keys_count BIGINT;
v_null_count BIGINT;
v_orphan_departure_count BIGINT;
v_orphan_arrival_count BIGINT;
v_orphan_airplane_count BIGINT;
BEGIN
-- Для snapshot-справочников пустой батч — ошибка.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.routes
WHERE batch_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.routes пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
-- В ODS не должно быть дублей по составному бизнес-ключу.
SELECT COUNT(*)
INTO v_dup_count
FROM (
SELECT route_no, validity
FROM ods.routes
GROUP BY route_no, validity
HAVING COUNT(*) > 1
) AS d;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes найдены дубликаты (route_no, validity): %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT route_no, validity
FROM stg.routes
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.routes AS o
WHERE o.route_no = s.route_no
AND o.validity = s.validity
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes отсутствуют ключи из stg.routes (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- В ODS не должно быть лишних ключей, которых нет в snapshot текущего батча.
SELECT COUNT(*)
INTO v_extra_keys_count
FROM ods.routes AS o
WHERE NOT EXISTS (
SELECT 1
FROM (
SELECT DISTINCT route_no, validity
FROM stg.routes
WHERE batch_id = v_batch_id
) AS s
WHERE s.route_no = o.route_no
AND s.validity = o.validity
);
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes найдены лишние ключи вне stg batch_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.routes
WHERE route_no IS NULL
OR route_no = ''
OR validity IS NULL
OR validity = ''
OR departure_airport IS NULL
OR departure_airport = ''
OR arrival_airport IS NULL
OR arrival_airport = ''
OR airplane_code IS NULL
OR airplane_code = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
-- Ссылочная целостность: departure_airport должен существовать в ods.airports.
SELECT COUNT(*)
INTO v_orphan_departure_count
FROM ods.routes AS r
WHERE NOT EXISTS (
SELECT 1
FROM ods.airports AS a
WHERE a.airport_code = r.departure_airport
);
IF v_orphan_departure_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes найдены строки с невалидным departure_airport: %',
v_orphan_departure_count;
END IF;
-- Ссылочная целостность: arrival_airport должен существовать в ods.airports.
SELECT COUNT(*)
INTO v_orphan_arrival_count
FROM ods.routes AS r
WHERE NOT EXISTS (
SELECT 1
FROM ods.airports AS a
WHERE a.airport_code = r.arrival_airport
);
IF v_orphan_arrival_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes найдены строки с невалидным arrival_airport: %',
v_orphan_arrival_count;
END IF;
-- Ссылочная целостность: airplane_code должен существовать в ods.airplanes.
SELECT COUNT(*)
INTO v_orphan_airplane_count
FROM ods.routes AS r
WHERE NOT EXISTS (
SELECT 1
FROM ods.airplanes AS a
WHERE a.airplane_code = r.airplane_code
);
IF v_orphan_airplane_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes найдены строки с невалидным airplane_code: %',
v_orphan_airplane_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.routes ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/routes_load.sql
+118
View File
@@ -0,0 +1,118 @@
-- Загрузка ODS по routes: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.route_no,
s.validity,
s.departure_airport,
s.arrival_airport,
s.airplane_code,
s.days_of_week,
NULLIF(s.scheduled_time, '')::TIME AS departure_time,
NULLIF(s.duration, '')::INTERVAL AS duration,
ROW_NUMBER() OVER (
PARTITION BY s.route_no, s.validity
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.routes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.routes AS o
SET departure_airport = s.departure_airport,
arrival_airport = s.arrival_airport,
airplane_code = s.airplane_code,
days_of_week = s.days_of_week,
departure_time = s.departure_time,
duration = s.duration,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.route_no = s.route_no
AND o.validity = s.validity
AND (
o.departure_airport IS DISTINCT FROM s.departure_airport
OR o.arrival_airport IS DISTINCT FROM s.arrival_airport
OR o.airplane_code IS DISTINCT FROM s.airplane_code
OR o.days_of_week IS DISTINCT FROM s.days_of_week
OR o.departure_time IS DISTINCT FROM s.departure_time
OR o.duration IS DISTINCT FROM s.duration
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.route_no,
s.validity,
s.departure_airport,
s.arrival_airport,
s.airplane_code,
s.days_of_week,
NULLIF(s.scheduled_time, '')::TIME AS departure_time,
NULLIF(s.duration, '')::INTERVAL AS duration,
ROW_NUMBER() OVER (
PARTITION BY s.route_no, s.validity
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.routes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.routes (
route_no,
validity,
departure_airport,
arrival_airport,
airplane_code,
days_of_week,
departure_time,
duration,
_load_id,
_load_ts
)
SELECT
s.route_no,
s.validity,
s.departure_airport,
s.arrival_airport,
s.airplane_code,
s.days_of_week,
s.departure_time,
s.duration,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.routes AS o
WHERE o.route_no = s.route_no
AND o.validity = s.validity
);
-- Statement 3: DELETE ключей, которых нет в snapshot текущего батча.
-- Это делает ODS для справочника действительно "current state".
WITH src_keys AS (
SELECT d.route_no, d.validity
FROM (
SELECT
s.route_no,
s.validity,
ROW_NUMBER() OVER (
PARTITION BY s.route_no, s.validity
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.routes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
) AS d
WHERE d.rn = 1
)
DELETE FROM ods.routes AS o
WHERE NOT EXISTS (
SELECT 1
FROM src_keys AS s
WHERE s.route_no = o.route_no
AND s.validity = o.validity
);
ANALYZE ods.routes;
sql/ods/seats_ddl.sql
+12
View File
@@ -0,0 +1,12 @@
-- DDL для ODS-слоя по таблице seats (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.seats (
airplane_code TEXT NOT NULL,
seat_no TEXT NOT NULL,
fare_conditions TEXT NOT NULL,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (airplane_code);
sql/ods/seats_dq.sql
+125
View File
@@ -0,0 +1,125 @@
-- DQ для ODS seats.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_extra_keys_count BIGINT;
v_null_count BIGINT;
v_orphan_airplane_count BIGINT;
BEGIN
-- Для snapshot-справочников пустой батч — ошибка.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.seats
WHERE batch_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.seats пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
-- В ODS не должно быть дублей по составному бизнес-ключу.
SELECT COUNT(*)
INTO v_dup_count
FROM (
SELECT airplane_code, seat_no
FROM ods.seats
GROUP BY airplane_code, seat_no
HAVING COUNT(*) > 1
) AS d;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.seats найдены дубликаты (airplane_code, seat_no): %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT airplane_code, seat_no
FROM stg.seats
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.seats AS o
WHERE o.airplane_code = s.airplane_code
AND o.seat_no = s.seat_no
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.seats отсутствуют ключи из stg.seats (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- В ODS не должно быть лишних ключей, которых нет в snapshot текущего батча.
SELECT COUNT(*)
INTO v_extra_keys_count
FROM ods.seats AS o
WHERE NOT EXISTS (
SELECT 1
FROM (
SELECT DISTINCT airplane_code, seat_no
FROM stg.seats
WHERE batch_id = v_batch_id
) AS s
WHERE s.airplane_code = o.airplane_code
AND s.seat_no = o.seat_no
);
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.seats найдены лишние ключи вне stg batch_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.seats
WHERE airplane_code IS NULL
OR airplane_code = ''
OR seat_no IS NULL
OR seat_no = ''
OR fare_conditions IS NULL
OR fare_conditions = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.seats найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
-- Ссылочная целостность: airplane_code должен существовать в ods.airplanes.
SELECT COUNT(*)
INTO v_orphan_airplane_count
FROM ods.seats AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.airplanes AS a
WHERE a.airplane_code = s.airplane_code
);
IF v_orphan_airplane_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.seats найдены строки с невалидным airplane_code: %',
v_orphan_airplane_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.seats ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/seats_load.sql
+86
View File
@@ -0,0 +1,86 @@
-- Загрузка ODS по seats: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.airplane_code,
s.seat_no,
s.fare_conditions,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code, s.seat_no
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.seats AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.seats AS o
SET fare_conditions = s.fare_conditions,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.airplane_code = s.airplane_code
AND o.seat_no = s.seat_no
AND o.fare_conditions IS DISTINCT FROM s.fare_conditions;
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.airplane_code,
s.seat_no,
s.fare_conditions,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code, s.seat_no
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.seats AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.seats (
airplane_code,
seat_no,
fare_conditions,
_load_id,
_load_ts
)
SELECT
s.airplane_code,
s.seat_no,
s.fare_conditions,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.seats AS o
WHERE o.airplane_code = s.airplane_code
AND o.seat_no = s.seat_no
);
-- Statement 3: DELETE ключей, которых нет в snapshot текущего батча.
-- Это делает ODS для справочника действительно "current state".
WITH src_keys AS (
SELECT d.airplane_code, d.seat_no
FROM (
SELECT
s.airplane_code,
s.seat_no,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code, s.seat_no
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
) AS rn
FROM stg.seats AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
) AS d
WHERE d.rn = 1
)
DELETE FROM ods.seats AS o
WHERE NOT EXISTS (
SELECT 1
FROM src_keys AS s
WHERE s.airplane_code = o.airplane_code
AND s.seat_no = o.seat_no
);
ANALYZE ods.seats;
sql/ods/segments_ddl.sql
+14
View File
@@ -0,0 +1,14 @@
-- DDL для ODS-слоя по таблице segments (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.segments (
ticket_no TEXT NOT NULL,
flight_id INTEGER NOT NULL,
fare_conditions TEXT NOT NULL,
segment_amount NUMERIC(10,2),
event_ts TIMESTAMP,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (ticket_no);
sql/ods/segments_dq.sql
+112
View File
@@ -0,0 +1,112 @@
-- DQ для ODS segments.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_null_count BIGINT;
v_orphan_ticket_count BIGINT;
v_orphan_flight_count BIGINT;
BEGIN
-- Для инкрементальных таблиц пустой батч допустим.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.segments
WHERE batch_id = v_batch_id;
-- В ODS не должно быть дублей по составному бизнес-ключу.
SELECT COUNT(*)
INTO v_dup_count
FROM (
SELECT ticket_no, flight_id
FROM ods.segments
GROUP BY ticket_no, flight_id
HAVING COUNT(*) > 1
) AS d;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.segments найдены дубликаты (ticket_no, flight_id): %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT ticket_no, NULLIF(flight_id, '')::INTEGER AS flight_id
FROM stg.segments
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.segments AS o
WHERE o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.segments отсутствуют ключи из stg.segments (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.segments
WHERE ticket_no IS NULL
OR ticket_no = ''
OR flight_id IS NULL
OR fare_conditions IS NULL
OR fare_conditions = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.segments найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
-- Ссылочная целостность: segments.ticket_no -> tickets.ticket_no.
SELECT COUNT(*)
INTO v_orphan_ticket_count
FROM ods.segments AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.tickets AS t
WHERE t.ticket_no = s.ticket_no
);
IF v_orphan_ticket_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.segments найдены строки без соответствующего tickets.ticket_no: %',
v_orphan_ticket_count;
END IF;
-- Ссылочная целостность: segments.flight_id -> flights.flight_id.
SELECT COUNT(*)
INTO v_orphan_flight_count
FROM ods.segments AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.flights AS f
WHERE f.flight_id = s.flight_id
);
IF v_orphan_flight_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.segments найдены строки без соответствующего flights.flight_id: %',
v_orphan_flight_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.segments ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/segments_load.sql
+75
View File
@@ -0,0 +1,75 @@
-- Загрузка ODS по segments: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.ticket_no,
NULLIF(s.flight_id, '')::INTEGER AS flight_id,
s.fare_conditions,
NULLIF(s.price, '')::NUMERIC(10,2) AS segment_amount,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no, s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.segments AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.segments AS o
SET fare_conditions = s.fare_conditions,
segment_amount = s.segment_amount,
event_ts = s.event_ts,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
AND (
o.fare_conditions IS DISTINCT FROM s.fare_conditions
OR o.segment_amount IS DISTINCT FROM s.segment_amount
OR o.event_ts IS DISTINCT FROM s.event_ts
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.ticket_no,
NULLIF(s.flight_id, '')::INTEGER AS flight_id,
s.fare_conditions,
NULLIF(s.price, '')::NUMERIC(10,2) AS segment_amount,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no, s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.segments AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.segments (
ticket_no,
flight_id,
fare_conditions,
segment_amount,
event_ts,
_load_id,
_load_ts
)
SELECT
s.ticket_no,
s.flight_id,
s.fare_conditions,
s.segment_amount,
s.event_ts,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.segments AS o
WHERE o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
);
ANALYZE ods.segments;
sql/ods/tickets_ddl.sql
+15
View File
@@ -0,0 +1,15 @@
-- DDL для ODS-слоя по таблице tickets (текущее состояние, SCD1).
CREATE SCHEMA IF NOT EXISTS ods;
CREATE TABLE IF NOT EXISTS ods.tickets (
ticket_no TEXT NOT NULL,
book_ref TEXT NOT NULL,
passenger_id TEXT NOT NULL,
passenger_name TEXT NOT NULL,
is_outbound BOOLEAN,
event_ts TIMESTAMP,
_load_id TEXT NOT NULL,
_load_ts TIMESTAMP NOT NULL DEFAULT now()
)
DISTRIBUTED BY (ticket_no);
sql/ods/tickets_dq.sql
+92
View File
@@ -0,0 +1,92 @@
-- DQ для ODS tickets.
DO $$
DECLARE
v_batch_id TEXT := '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text;
v_stg_batch_count BIGINT;
v_dup_count BIGINT;
v_missing_keys_count BIGINT;
v_null_count BIGINT;
v_orphan_booking_count BIGINT;
BEGIN
-- Для инкрементальных таблиц пустой батч допустим.
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.tickets
WHERE batch_id = v_batch_id;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT ticket_no)
INTO v_dup_count
FROM ods.tickets;
IF v_dup_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.tickets найдены дубликаты ticket_no: %',
v_dup_count;
END IF;
-- Все ключи из STG текущего батча должны присутствовать в ODS.
SELECT COUNT(*)
INTO v_missing_keys_count
FROM (
SELECT DISTINCT ticket_no
FROM stg.tickets
WHERE batch_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
FROM ods.tickets AS o
WHERE o.ticket_no = s.ticket_no
);
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.tickets отсутствуют ключи из stg.tickets (batch_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
-- Обязательные поля в ODS.
SELECT COUNT(*)
INTO v_null_count
FROM ods.tickets
WHERE ticket_no IS NULL
OR ticket_no = ''
OR book_ref IS NULL
OR book_ref = ''
OR passenger_id IS NULL
OR passenger_id = ''
OR passenger_name IS NULL
OR passenger_name = ''
OR _load_id IS NULL
OR _load_id = ''
OR _load_ts IS NULL;
IF v_null_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.tickets найдены NULL/пустые обязательные поля: %',
v_null_count;
END IF;
-- Ссылочная целостность: tickets.book_ref -> bookings.book_ref.
SELECT COUNT(*)
INTO v_orphan_booking_count
FROM ods.tickets AS t
WHERE NOT EXISTS (
SELECT 1
FROM ods.bookings AS b
WHERE b.book_ref = t.book_ref
);
IF v_orphan_booking_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.tickets найдены строки без соответствующего bookings.book_ref: %',
v_orphan_booking_count;
END IF;
RAISE NOTICE
'DQ PASSED: ods.tickets ок (batch_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/tickets_load.sql
+81
View File
@@ -0,0 +1,81 @@
-- Загрузка ODS по tickets: SCD1 (UPDATE изменившихся + INSERT новых).
-- Statement 1: UPDATE существующих строк.
WITH src AS (
SELECT
s.ticket_no,
s.book_ref,
s.passenger_id,
s.passenger_name,
NULLIF(s.outbound, '')::BOOLEAN AS is_outbound,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.tickets AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
UPDATE ods.tickets AS o
SET book_ref = s.book_ref,
passenger_id = s.passenger_id,
passenger_name = s.passenger_name,
is_outbound = s.is_outbound,
event_ts = s.event_ts,
_load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
_load_ts = now()
FROM src AS s
WHERE s.rn = 1
AND o.ticket_no = s.ticket_no
AND (
o.book_ref IS DISTINCT FROM s.book_ref
OR o.passenger_id IS DISTINCT FROM s.passenger_id
OR o.passenger_name IS DISTINCT FROM s.passenger_name
OR o.is_outbound IS DISTINCT FROM s.is_outbound
OR o.event_ts IS DISTINCT FROM s.event_ts
);
-- Statement 2: INSERT новых строк.
WITH src AS (
SELECT
s.ticket_no,
s.book_ref,
s.passenger_id,
s.passenger_name,
NULLIF(s.outbound, '')::BOOLEAN AS is_outbound,
s.src_created_at_ts AS event_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
) AS rn
FROM stg.tickets AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
INSERT INTO ods.tickets (
ticket_no,
book_ref,
passenger_id,
passenger_name,
is_outbound,
event_ts,
_load_id,
_load_ts
)
SELECT
s.ticket_no,
s.book_ref,
s.passenger_id,
s.passenger_name,
s.is_outbound,
s.event_ts,
'{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text,
now()
FROM src AS s
WHERE s.rn = 1
AND NOT EXISTS (
SELECT 1
FROM ods.tickets AS o
WHERE o.ticket_no = s.ticket_no
);
ANALYZE ods.tickets;