ddmitry/airflow-greenplum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(all): унифицированы служебные поля STG — переход на канонический нейминг

Browse Source 
- Зачем:
  - STG-слой использовал legacy-имена (batch_id, load_dttm, src_created_at_ts),
    тогда как ODS/DDS/DM уже работали с каноном (_load_id, _load_ts, event_ts).
    Студент видел разные имена для одного понятия — это убрано.
- Что:
  - переименованы колонки в 9 STG DDL: batch_id→_load_id, load_dttm→_load_ts,
    src_created_at_ts→event_ts; добавлен NOT NULL для _load_id во всех таблицах.
  - обновлены 9 STG Load, 9 STG DQ, 9 ODS Load, 9 ODS DQ (INSERT/SELECT/WHERE).
  - обновлены DAG-файлы bookings_to_gp_stage.py и bookings_to_gp_ods.py
    (встроенный SQL резолвера, комментарии; Python-идентификаторы не тронуты).
  - обновлены тесты и ~15 документов (naming_conventions, PRD, db_schema,
    design-docs, qa-plan, README, TESTING и др.).
- Проверка:
  - grep -rn 'load_dttm\|src_created_at_ts' sql/ airflow/ tests/ — 0 совпадений.
  - make test — 4 passed.
  - e2e-etl: day1 прошёл полностью, day2 стартовал без ошибок.
This commit is contained in:
ddadmin
2026-03-10 00:07:08 +03:00
parent 3de4db8aa6
commit 5972bcc2d8
64 changed files with 582 additions and 484 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sql/ods/airplanes_dq.sql
+7 -7
View File
@@ -13,11 +13,11 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.airplanes
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.airplanes пустой. Проверьте загрузку STG и PXF.',
'DQ FAILED: _load_id=% для stg.airplanes пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
@@ -38,7 +38,7 @@ BEGIN
FROM (
SELECT DISTINCT airplane_code
FROM stg.airplanes
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -48,7 +48,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airplanes отсутствуют ключи из stg.airplanes (batch_id=%): %',
'DQ FAILED: в ods.airplanes отсутствуют ключи из stg.airplanes (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -62,14 +62,14 @@ BEGIN
FROM (
SELECT DISTINCT airplane_code
FROM stg.airplanes
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE s.airplane_code = o.airplane_code
);
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airplanes найдены лишние ключи вне stg batch_id=%: %',
'DQ FAILED: в ods.airplanes найдены лишние ключи вне stg _load_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
@@ -93,7 +93,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.airplanes ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.airplanes ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/airplanes_load.sql
+2 -2
View File
@@ -21,10 +21,10 @@ WITH src AS (
NULLIF(s.speed, '')::INTEGER AS speed_kmh,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
ORDER BY s._load_ts DESC, s.event_ts DESC NULLS LAST
) AS rn
FROM stg.airplanes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
WHERE s._load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
SELECT
s.airplane_code,
sql/ods/airports_dq.sql
+7 -7
View File
@@ -13,11 +13,11 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.airports
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.airports пустой. Проверьте загрузку STG и PXF.',
'DQ FAILED: _load_id=% для stg.airports пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
@@ -38,7 +38,7 @@ BEGIN
FROM (
SELECT DISTINCT airport_code
FROM stg.airports
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -48,7 +48,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airports отсутствуют ключи из stg.airports (batch_id=%): %',
'DQ FAILED: в ods.airports отсутствуют ключи из stg.airports (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -62,14 +62,14 @@ BEGIN
FROM (
SELECT DISTINCT airport_code
FROM stg.airports
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE s.airport_code = o.airport_code
);
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.airports найдены лишние ключи вне stg batch_id=%: %',
'DQ FAILED: в ods.airports найдены лишние ключи вне stg _load_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
@@ -99,7 +99,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.airports ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.airports ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/airports_load.sql
+2 -2
View File
@@ -25,10 +25,10 @@ WITH src AS (
s.timezone,
ROW_NUMBER() OVER (
PARTITION BY s.airport_code
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
ORDER BY s._load_ts DESC, s.event_ts DESC NULLS LAST
) AS rn
FROM stg.airports AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
WHERE s._load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
SELECT
s.airport_code,
sql/ods/boarding_passes_dq.sql
+4 -4
View File
@@ -13,7 +13,7 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.boarding_passes
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
-- В ODS не должно быть дублей по составному бизнес-ключу.
SELECT COUNT(*)
@@ -37,7 +37,7 @@ BEGIN
FROM (
SELECT DISTINCT ticket_no, NULLIF(flight_id, '')::INTEGER AS flight_id
FROM stg.boarding_passes
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -48,7 +48,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.boarding_passes отсутствуют ключи из stg.boarding_passes (batch_id=%): %',
'DQ FAILED: в ods.boarding_passes отсутствуют ключи из stg.boarding_passes (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -90,7 +90,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.boarding_passes ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.boarding_passes ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/boarding_passes_load.sql
+9 -9
View File
@@ -10,16 +10,16 @@ WITH src AS (
s.seat_no,
NULLIF(s.boarding_no, '')::INTEGER AS boarding_no,
NULLIF(s.boarding_time, '')::TIMESTAMP WITH TIME ZONE AS boarding_time,
s.src_created_at_ts AS event_ts,
s.batch_id,
s.load_dttm,
s.event_ts,
s._load_id,
s._load_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no, s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
ORDER BY s.event_ts DESC NULLS LAST, s._load_ts DESC
) AS rn
FROM stg.boarding_passes AS s
-- Используем HWM (High Water Mark) по техническому времени STG
WHERE s.load_dttm > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.boarding_passes)
WHERE s._load_ts > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.boarding_passes)
)
SELECT * FROM src WHERE rn = 1;
@@ -29,8 +29,8 @@ SET seat_no = s.seat_no,
boarding_no = s.boarding_no,
boarding_time = s.boarding_time,
event_ts = s.event_ts,
_load_id = s.batch_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s.load_dttm -- Фиксируем время STG как водяной знак для ODS
_load_id = s._load_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s._load_ts -- Фиксируем время STG как водяной знак для ODS
FROM tmp_boarding_passes_delta AS s
WHERE o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
@@ -59,8 +59,8 @@ SELECT
s.boarding_no,
s.boarding_time,
s.event_ts,
s.batch_id,
s.load_dttm
s._load_id,
s._load_ts
FROM tmp_boarding_passes_delta AS s
WHERE NOT EXISTS (
SELECT 1
sql/ods/bookings_dq.sql
+4 -4
View File
@@ -12,7 +12,7 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.bookings
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT book_ref)
@@ -31,7 +31,7 @@ BEGIN
FROM (
SELECT DISTINCT book_ref
FROM stg.bookings
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -41,7 +41,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.bookings отсутствуют ключи из stg.bookings (batch_id=%): %',
'DQ FAILED: в ods.bookings отсутствуют ключи из stg.bookings (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -65,7 +65,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.bookings ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.bookings ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/bookings_load.sql
+9 -9
View File
@@ -8,16 +8,16 @@ WITH src AS (
s.book_ref,
NULLIF(s.book_date, '')::TIMESTAMP WITH TIME ZONE AS book_date,
NULLIF(s.total_amount, '')::NUMERIC(10,2) AS total_amount,
s.src_created_at_ts AS event_ts,
s.batch_id,
s.load_dttm,
s.event_ts,
s._load_id,
s._load_ts,
ROW_NUMBER() OVER (
PARTITION BY s.book_ref
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
ORDER BY s.event_ts DESC NULLS LAST, s._load_ts DESC
) AS rn
FROM stg.bookings AS s
-- Используем HWM (High Water Mark) по техническому времени STG
WHERE s.load_dttm > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.bookings)
WHERE s._load_ts > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.bookings)
)
SELECT * FROM src WHERE rn = 1;
@@ -26,8 +26,8 @@ UPDATE ods.bookings AS o
SET book_date = s.book_date,
total_amount = s.total_amount,
event_ts = s.event_ts,
_load_id = s.batch_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s.load_dttm -- Фиксируем время STG как водяной знак для ODS
_load_id = s._load_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s._load_ts -- Фиксируем время STG как водяной знак для ODS
FROM tmp_bookings_delta AS s
WHERE o.book_ref = s.book_ref
AND (
@@ -50,8 +50,8 @@ SELECT
s.book_date,
s.total_amount,
s.event_ts,
s.batch_id,
s.load_dttm
s._load_id,
s._load_ts
FROM tmp_bookings_delta AS s
WHERE NOT EXISTS (
SELECT 1
sql/ods/flights_dq.sql
+4 -4
View File
@@ -13,7 +13,7 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.flights
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT flight_id)
@@ -32,7 +32,7 @@ BEGIN
FROM (
SELECT DISTINCT NULLIF(flight_id, '')::INTEGER AS flight_id
FROM stg.flights
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -42,7 +42,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.flights отсутствуют ключи из stg.flights (batch_id=%): %',
'DQ FAILED: в ods.flights отсутствуют ключи из stg.flights (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -83,7 +83,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.flights ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.flights ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/flights_load.sql
+16 -16
View File
@@ -9,7 +9,7 @@ WITH segment_flights AS (
SELECT DISTINCT
NULLIF(s.flight_id, '')::INTEGER AS flight_id
FROM stg.segments AS s
WHERE s.load_dttm > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.segments)
WHERE s._load_ts > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.segments)
AND s.flight_id IS NOT NULL
AND s.flight_id <> ''
),
@@ -22,9 +22,9 @@ stg_flights_typed AS (
NULLIF(s.scheduled_arrival, '')::TIMESTAMP WITH TIME ZONE AS scheduled_arrival,
NULLIF(s.actual_departure, '')::TIMESTAMP WITH TIME ZONE AS actual_departure,
NULLIF(s.actual_arrival, '')::TIMESTAMP WITH TIME ZONE AS actual_arrival,
s.src_created_at_ts AS event_ts,
s.load_dttm,
s.batch_id
s.event_ts,
s._load_ts,
s._load_id
FROM stg.flights AS s
WHERE s.flight_id IS NOT NULL
AND s.flight_id <> ''
@@ -40,10 +40,10 @@ src_union AS (
f.actual_departure,
f.actual_arrival,
f.event_ts,
f.load_dttm,
f.batch_id
f._load_ts,
f._load_id
FROM stg_flights_typed AS f
WHERE f.load_dttm > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.flights)
WHERE f._load_ts > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.flights)
UNION ALL
@@ -57,8 +57,8 @@ src_union AS (
f.actual_departure,
f.actual_arrival,
f.event_ts,
f.load_dttm,
f.batch_id
f._load_ts,
f._load_id
FROM stg_flights_typed AS f
JOIN segment_flights AS sf
ON sf.flight_id = f.flight_id
@@ -73,11 +73,11 @@ src AS (
u.actual_departure,
u.actual_arrival,
u.event_ts,
u.batch_id,
u.load_dttm,
u._load_id,
u._load_ts,
ROW_NUMBER() OVER (
PARTITION BY u.flight_id
ORDER BY u.event_ts DESC NULLS LAST, u.load_dttm DESC
ORDER BY u.event_ts DESC NULLS LAST, u._load_ts DESC
) AS rn
FROM src_union AS u
)
@@ -92,8 +92,8 @@ SET route_no = s.route_no,
actual_departure = s.actual_departure,
actual_arrival = s.actual_arrival,
event_ts = s.event_ts,
_load_id = s.batch_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s.load_dttm -- Фиксируем время STG как водяной знак для ODS
_load_id = s._load_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s._load_ts -- Фиксируем время STG как водяной знак для ODS
FROM tmp_flights_delta AS s
WHERE o.flight_id = s.flight_id
AND (
@@ -128,8 +128,8 @@ SELECT
s.actual_departure,
s.actual_arrival,
s.event_ts,
s.batch_id,
s.load_dttm
s._load_id,
s._load_ts
FROM tmp_flights_delta AS s
WHERE NOT EXISTS (
SELECT 1
sql/ods/routes_dq.sql
+7 -7
View File
@@ -16,11 +16,11 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.routes
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.routes пустой. Проверьте загрузку STG и PXF.',
'DQ FAILED: _load_id=% для stg.routes пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
@@ -46,7 +46,7 @@ BEGIN
FROM (
SELECT DISTINCT route_no, validity
FROM stg.routes
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -57,7 +57,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes отсутствуют ключи из stg.routes (batch_id=%): %',
'DQ FAILED: в ods.routes отсутствуют ключи из stg.routes (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -71,7 +71,7 @@ BEGIN
FROM (
SELECT DISTINCT route_no, validity
FROM stg.routes
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE s.route_no = o.route_no
AND s.validity = o.validity
@@ -79,7 +79,7 @@ BEGIN
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.routes найдены лишние ключи вне stg batch_id=%: %',
'DQ FAILED: в ods.routes найдены лишние ключи вне stg _load_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
@@ -157,7 +157,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.routes ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.routes ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/routes_load.sql
+2 -2
View File
@@ -29,10 +29,10 @@ WITH src AS (
NULLIF(s.duration, '')::INTERVAL AS duration,
ROW_NUMBER() OVER (
PARTITION BY s.route_no, s.validity
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
ORDER BY s._load_ts DESC, s.event_ts DESC NULLS LAST
) AS rn
FROM stg.routes AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
WHERE s._load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
SELECT
s.route_no,
sql/ods/seats_dq.sql
+7 -7
View File
@@ -14,11 +14,11 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.seats
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
IF v_stg_batch_count = 0 THEN
RAISE EXCEPTION
'DQ FAILED: batch_id=% для stg.seats пустой. Проверьте загрузку STG и PXF.',
'DQ FAILED: _load_id=% для stg.seats пустой. Проверьте загрузку STG и PXF.',
v_batch_id;
END IF;
@@ -44,7 +44,7 @@ BEGIN
FROM (
SELECT DISTINCT airplane_code, seat_no
FROM stg.seats
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -55,7 +55,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.seats отсутствуют ключи из stg.seats (batch_id=%): %',
'DQ FAILED: в ods.seats отсутствуют ключи из stg.seats (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -69,7 +69,7 @@ BEGIN
FROM (
SELECT DISTINCT airplane_code, seat_no
FROM stg.seats
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE s.airplane_code = o.airplane_code
AND s.seat_no = o.seat_no
@@ -77,7 +77,7 @@ BEGIN
IF v_extra_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.seats найдены лишние ключи вне stg batch_id=%: %',
'DQ FAILED: в ods.seats найдены лишние ключи вне stg _load_id=%: %',
v_batch_id,
v_extra_keys_count;
END IF;
@@ -119,7 +119,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.seats ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.seats ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/seats_load.sql
+2 -2
View File
@@ -19,10 +19,10 @@ WITH src AS (
s.fare_conditions,
ROW_NUMBER() OVER (
PARTITION BY s.airplane_code, s.seat_no
ORDER BY s.load_dttm DESC, s.src_created_at_ts DESC NULLS LAST
ORDER BY s._load_ts DESC, s.event_ts DESC NULLS LAST
) AS rn
FROM stg.seats AS s
WHERE s.batch_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
WHERE s._load_id = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'::text
)
SELECT
s.airplane_code,
sql/ods/segments_dq.sql
+4 -4
View File
@@ -14,7 +14,7 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.segments
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
-- В ODS не должно быть дублей по составному бизнес-ключу.
SELECT COUNT(*)
@@ -38,7 +38,7 @@ BEGIN
FROM (
SELECT DISTINCT ticket_no, NULLIF(flight_id, '')::INTEGER AS flight_id
FROM stg.segments
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -49,7 +49,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.segments отсутствуют ключи из stg.segments (batch_id=%): %',
'DQ FAILED: в ods.segments отсутствуют ключи из stg.segments (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -106,7 +106,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.segments ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.segments ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/segments_load.sql
+9 -9
View File
@@ -9,16 +9,16 @@ WITH src AS (
NULLIF(s.flight_id, '')::INTEGER AS flight_id,
s.fare_conditions,
NULLIF(s.price, '')::NUMERIC(10,2) AS amount,
s.src_created_at_ts AS event_ts,
s.batch_id,
s.load_dttm,
s.event_ts,
s._load_id,
s._load_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no, s.flight_id
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
ORDER BY s.event_ts DESC NULLS LAST, s._load_ts DESC
) AS rn
FROM stg.segments AS s
-- Используем HWM (High Water Mark) по техническому времени STG
WHERE s.load_dttm > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.segments)
WHERE s._load_ts > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.segments)
)
SELECT * FROM src WHERE rn = 1;
@@ -27,8 +27,8 @@ UPDATE ods.segments AS o
SET fare_conditions = s.fare_conditions,
amount = s.amount,
event_ts = s.event_ts,
_load_id = s.batch_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s.load_dttm -- Фиксируем время STG как водяной знак для ODS
_load_id = s._load_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s._load_ts -- Фиксируем время STG как водяной знак для ODS
FROM tmp_segments_delta AS s
WHERE o.ticket_no = s.ticket_no
AND o.flight_id = s.flight_id
@@ -54,8 +54,8 @@ SELECT
s.fare_conditions,
s.amount,
s.event_ts,
s.batch_id,
s.load_dttm
s._load_id,
s._load_ts
FROM tmp_segments_delta AS s
WHERE NOT EXISTS (
SELECT 1
sql/ods/tickets_dq.sql
+4 -4
View File
@@ -13,7 +13,7 @@ BEGIN
SELECT COUNT(*)
INTO v_stg_batch_count
FROM stg.tickets
WHERE batch_id = v_batch_id;
WHERE _load_id = v_batch_id;
-- В ODS не должно быть дублей по бизнес-ключу.
SELECT COUNT(*) - COUNT(DISTINCT ticket_no)
@@ -32,7 +32,7 @@ BEGIN
FROM (
SELECT DISTINCT ticket_no
FROM stg.tickets
WHERE batch_id = v_batch_id
WHERE _load_id = v_batch_id
) AS s
WHERE NOT EXISTS (
SELECT 1
@@ -42,7 +42,7 @@ BEGIN
IF v_missing_keys_count <> 0 THEN
RAISE EXCEPTION
'DQ FAILED: в ods.tickets отсутствуют ключи из stg.tickets (batch_id=%): %',
'DQ FAILED: в ods.tickets отсутствуют ключи из stg.tickets (_load_id=%): %',
v_batch_id,
v_missing_keys_count;
END IF;
@@ -86,7 +86,7 @@ BEGIN
END IF;
RAISE NOTICE
'DQ PASSED: ods.tickets ок (batch_id=%): stg_batch_rows=%',
'DQ PASSED: ods.tickets ок (_load_id=%): stg_batch_rows=%',
v_batch_id,
v_stg_batch_count;
END $$;
sql/ods/tickets_load.sql
+9 -9
View File
@@ -10,16 +10,16 @@ WITH src AS (
s.passenger_id,
s.passenger_name,
NULLIF(s.outbound, '')::BOOLEAN AS is_outbound,
s.src_created_at_ts AS event_ts,
s.batch_id,
s.load_dttm,
s.event_ts,
s._load_id,
s._load_ts,
ROW_NUMBER() OVER (
PARTITION BY s.ticket_no
ORDER BY s.src_created_at_ts DESC NULLS LAST, s.load_dttm DESC
ORDER BY s.event_ts DESC NULLS LAST, s._load_ts DESC
) AS rn
FROM stg.tickets AS s
-- Используем HWM (High Water Mark) по техническому времени STG
WHERE s.load_dttm > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.tickets)
WHERE s._load_ts > (SELECT COALESCE(MAX(_load_ts), '1900-01-01 00:00:00'::TIMESTAMP) FROM ods.tickets)
)
SELECT * FROM src WHERE rn = 1;
@@ -30,8 +30,8 @@ SET book_ref = s.book_ref,
passenger_name = s.passenger_name,
is_outbound = s.is_outbound,
event_ts = s.event_ts,
_load_id = s.batch_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s.load_dttm -- Фиксируем время STG как водяной знак для ODS
_load_id = s._load_id, -- Сохраняем оригинальный lineage из STG
_load_ts = s._load_ts -- Фиксируем время STG как водяной знак для ODS
FROM tmp_tickets_delta AS s
WHERE o.ticket_no = s.ticket_no
AND (
@@ -60,8 +60,8 @@ SELECT
s.passenger_name,
s.is_outbound,
s.event_ts,
s.batch_id,
s.load_dttm
s._load_id,
s._load_ts
FROM tmp_tickets_delta AS s
WHERE NOT EXISTS (
SELECT 1