feat(main): заглушки студенческих файлов, ослабление DQ, адаптация тестов
- fact_flight_sales_dq: student SK (passenger/route/airplane) → NOTICE, airport_sk → порог 1% (эталон через ods.routes) - routes_dq: закомментирован RI airplane_code → ods.airplanes - ODS DAG: убрана зависимость dq_ods_airplanes → load_ods_routes - 18 студенческих файлов заменены заглушками (ODS/DDS/DM load+dq) - test_ods_sql_contract: SNAPSHOT_ENTITIES без airplanes/seats - test_dags_smoke: убран assert dq_ods_airplanes → dq_ods_routes Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -1,83 +1,3 @@
|
||||
-- DQ для DDS dim_airplanes.
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
v_row_count BIGINT;
|
||||
v_dup_sk BIGINT;
|
||||
v_dup_bk BIGINT;
|
||||
v_missing_bk BIGINT;
|
||||
v_null_count BIGINT;
|
||||
BEGIN
|
||||
-- Таблица не пуста.
|
||||
SELECT COUNT(*)
|
||||
INTO v_row_count
|
||||
FROM dds.dim_airplanes;
|
||||
|
||||
IF v_row_count = 0 THEN
|
||||
RAISE EXCEPTION 'DQ FAILED: dds.dim_airplanes пуста.';
|
||||
END IF;
|
||||
|
||||
-- Нет дублей по SK.
|
||||
SELECT COUNT(*) - COUNT(DISTINCT airplane_sk)
|
||||
INTO v_dup_sk
|
||||
FROM dds.dim_airplanes;
|
||||
|
||||
IF v_dup_sk <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_airplanes найдены дубликаты airplane_sk: %',
|
||||
v_dup_sk;
|
||||
END IF;
|
||||
|
||||
-- Нет дублей по BK.
|
||||
SELECT COUNT(*) - COUNT(DISTINCT airplane_bk)
|
||||
INTO v_dup_bk
|
||||
FROM dds.dim_airplanes;
|
||||
|
||||
IF v_dup_bk <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_airplanes найдены дубликаты airplane_bk: %',
|
||||
v_dup_bk;
|
||||
END IF;
|
||||
|
||||
-- Покрытие ODS: все airplane_code из ODS есть в DDS.
|
||||
SELECT COUNT(*)
|
||||
INTO v_missing_bk
|
||||
FROM (SELECT DISTINCT airplane_code FROM ods.airplanes) AS s
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM dds.dim_airplanes AS d
|
||||
WHERE d.airplane_bk = s.airplane_code
|
||||
);
|
||||
|
||||
IF v_missing_bk <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_airplanes отсутствуют ключи из ods.airplanes: %',
|
||||
v_missing_bk;
|
||||
END IF;
|
||||
|
||||
-- Обязательные поля.
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_count
|
||||
FROM dds.dim_airplanes
|
||||
WHERE airplane_sk IS NULL
|
||||
OR airplane_bk IS NULL
|
||||
OR airplane_bk = ''
|
||||
OR model IS NULL
|
||||
OR model = ''
|
||||
OR total_seats IS NULL
|
||||
OR created_at IS NULL
|
||||
OR updated_at IS NULL
|
||||
OR _load_id IS NULL
|
||||
OR _load_id = ''
|
||||
OR _load_ts IS NULL;
|
||||
|
||||
IF v_null_count <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_airplanes найдены NULL/пустые обязательные поля: %',
|
||||
v_null_count;
|
||||
END IF;
|
||||
|
||||
RAISE NOTICE
|
||||
'DQ PASSED: dds.dim_airplanes ок, строк=%',
|
||||
v_row_count;
|
||||
END $$;
|
||||
-- TODO: реализуйте проверки качества данных для dds.dim_airplanes
|
||||
-- Эталонную реализацию можно найти в ветке solution.
|
||||
SELECT 1;
|
||||
|
||||
@@ -1,96 +1,4 @@
|
||||
-- Загрузка DDS dim_airplanes: SCD1 UPSERT (UPDATE изменившихся + INSERT новых).
|
||||
|
||||
-- Statement 1: UPDATE существующих записей (если атрибуты изменились).
|
||||
WITH seats_agg AS (
|
||||
SELECT
|
||||
s.airplane_code,
|
||||
COUNT(*)::INTEGER AS total_seats
|
||||
FROM ods.seats AS s
|
||||
GROUP BY s.airplane_code
|
||||
),
|
||||
src AS (
|
||||
SELECT
|
||||
a.airplane_code,
|
||||
a.model,
|
||||
a.range_km,
|
||||
a.speed_kmh,
|
||||
COALESCE(sa.total_seats, 0) AS total_seats
|
||||
FROM ods.airplanes AS a
|
||||
LEFT JOIN seats_agg AS sa
|
||||
ON sa.airplane_code = a.airplane_code
|
||||
)
|
||||
UPDATE dds.dim_airplanes AS d
|
||||
SET model = s.model,
|
||||
range_km = s.range_km,
|
||||
speed_kmh = s.speed_kmh,
|
||||
total_seats = s.total_seats,
|
||||
updated_at = now(),
|
||||
_load_id = '{{ run_id }}',
|
||||
_load_ts = now()
|
||||
FROM src AS s
|
||||
WHERE d.airplane_bk = s.airplane_code
|
||||
AND (
|
||||
d.model IS DISTINCT FROM s.model
|
||||
OR d.range_km IS DISTINCT FROM s.range_km
|
||||
OR d.speed_kmh IS DISTINCT FROM s.speed_kmh
|
||||
OR d.total_seats IS DISTINCT FROM s.total_seats
|
||||
);
|
||||
|
||||
-- Statement 2: INSERT новых записей (MAX(sk) + ROW_NUMBER()).
|
||||
WITH seats_agg AS (
|
||||
SELECT
|
||||
s.airplane_code,
|
||||
COUNT(*)::INTEGER AS total_seats
|
||||
FROM ods.seats AS s
|
||||
GROUP BY s.airplane_code
|
||||
),
|
||||
src AS (
|
||||
SELECT
|
||||
a.airplane_code,
|
||||
a.model,
|
||||
a.range_km,
|
||||
a.speed_kmh,
|
||||
COALESCE(sa.total_seats, 0) AS total_seats
|
||||
FROM ods.airplanes AS a
|
||||
LEFT JOIN seats_agg AS sa
|
||||
ON sa.airplane_code = a.airplane_code
|
||||
),
|
||||
-- Учебный комментарий: Генерация SK через MAX() + ROW_NUMBER()
|
||||
-- Этот подход работает безопасно только потому, что Airflow запускает
|
||||
-- джобы загрузки для одной таблицы строго последовательно (concurrency=1).
|
||||
-- При параллельной загрузке возникнет состояние гонки (race condition) и возможны дубли SK.
|
||||
max_sk AS (
|
||||
SELECT COALESCE(MAX(airplane_sk), 0) AS v
|
||||
FROM dds.dim_airplanes
|
||||
)
|
||||
INSERT INTO dds.dim_airplanes (
|
||||
airplane_sk,
|
||||
airplane_bk,
|
||||
model,
|
||||
range_km,
|
||||
speed_kmh,
|
||||
total_seats,
|
||||
created_at,
|
||||
updated_at,
|
||||
_load_id,
|
||||
_load_ts
|
||||
)
|
||||
SELECT
|
||||
(SELECT v FROM max_sk) + ROW_NUMBER() OVER (ORDER BY s.airplane_code)::INTEGER,
|
||||
s.airplane_code,
|
||||
s.model,
|
||||
s.range_km,
|
||||
s.speed_kmh,
|
||||
s.total_seats,
|
||||
now(),
|
||||
now(),
|
||||
'{{ run_id }}',
|
||||
now()
|
||||
FROM src AS s
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM dds.dim_airplanes AS d
|
||||
WHERE d.airplane_bk = s.airplane_code
|
||||
);
|
||||
|
||||
ANALYZE dds.dim_airplanes;
|
||||
-- TODO: реализуйте загрузку dds.dim_airplanes (см. ТЗ в docs/assignment/analyst_spec.md)
|
||||
-- Паттерн: SCD1 UPSERT (UPDATE изменившихся + INSERT новых).
|
||||
-- Эталонную реализацию можно найти в ветке solution.
|
||||
SELECT 1;
|
||||
|
||||
@@ -1,100 +1,3 @@
|
||||
-- DQ для DDS dim_passengers.
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
v_row_count BIGINT;
|
||||
v_src_count BIGINT;
|
||||
v_dup_sk BIGINT;
|
||||
v_dup_bk BIGINT;
|
||||
v_missing_bk BIGINT;
|
||||
v_null_count BIGINT;
|
||||
BEGIN
|
||||
-- Для инкрементальных периодов без новых билетов допускаем пустую dim_passengers.
|
||||
SELECT COUNT(*)
|
||||
INTO v_row_count
|
||||
FROM dds.dim_passengers;
|
||||
|
||||
SELECT COUNT(DISTINCT passenger_id)
|
||||
INTO v_src_count
|
||||
FROM ods.tickets
|
||||
WHERE passenger_id IS NOT NULL
|
||||
AND passenger_id <> '';
|
||||
|
||||
IF v_row_count = 0 AND v_src_count <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: dds.dim_passengers пуста при непустом источнике ods.tickets (passenger_id=%).',
|
||||
v_src_count;
|
||||
ELSIF v_row_count = 0 AND v_src_count = 0 THEN
|
||||
RAISE NOTICE
|
||||
'DQ PASSED: dds.dim_passengers пуста, т.к. в ods.tickets нет passenger_id для загрузки.';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
-- Нет дублей по SK.
|
||||
SELECT COUNT(*) - COUNT(DISTINCT passenger_sk)
|
||||
INTO v_dup_sk
|
||||
FROM dds.dim_passengers;
|
||||
|
||||
IF v_dup_sk <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_passengers найдены дубликаты passenger_sk: %',
|
||||
v_dup_sk;
|
||||
END IF;
|
||||
|
||||
-- Нет дублей по BK (ID).
|
||||
SELECT COUNT(*) - COUNT(DISTINCT passenger_id)
|
||||
INTO v_dup_bk
|
||||
FROM dds.dim_passengers;
|
||||
|
||||
IF v_dup_bk <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_passengers найдены дубликаты passenger_id: %',
|
||||
v_dup_bk;
|
||||
END IF;
|
||||
|
||||
-- Покрытие ODS: все passenger_id из ods.tickets есть в DDS.
|
||||
SELECT COUNT(*)
|
||||
INTO v_missing_bk
|
||||
FROM (
|
||||
SELECT DISTINCT passenger_id
|
||||
FROM ods.tickets
|
||||
WHERE passenger_id IS NOT NULL
|
||||
AND passenger_id <> ''
|
||||
) AS s
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM dds.dim_passengers AS d
|
||||
WHERE d.passenger_id = s.passenger_id
|
||||
);
|
||||
|
||||
IF v_missing_bk <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_passengers отсутствуют passenger_id из ods.tickets: %',
|
||||
v_missing_bk;
|
||||
END IF;
|
||||
|
||||
-- Обязательные поля.
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_count
|
||||
FROM dds.dim_passengers
|
||||
WHERE passenger_sk IS NULL
|
||||
OR passenger_id IS NULL
|
||||
OR passenger_id = ''
|
||||
OR passenger_name IS NULL
|
||||
OR passenger_name = ''
|
||||
OR created_at IS NULL
|
||||
OR updated_at IS NULL
|
||||
OR _load_id IS NULL
|
||||
OR _load_id = ''
|
||||
OR _load_ts IS NULL;
|
||||
|
||||
IF v_null_count <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_passengers найдены NULL/пустые обязательные поля: %',
|
||||
v_null_count;
|
||||
END IF;
|
||||
|
||||
RAISE NOTICE
|
||||
'DQ PASSED: dds.dim_passengers ок, строк=%',
|
||||
v_row_count;
|
||||
END $$;
|
||||
-- TODO: реализуйте проверки качества данных для dds.dim_passengers
|
||||
-- Эталонную реализацию можно найти в ветке solution.
|
||||
SELECT 1;
|
||||
|
||||
@@ -1,67 +1,4 @@
|
||||
-- Загрузка DDS dim_passengers: SCD1 UPSERT (UPDATE изменившихся + INSERT новых).
|
||||
|
||||
-- Учебный комментарий: Используем TEMP TABLE для подготовки дельты.
|
||||
-- Это избавляет от дублирования сложной оконной функции в UPDATE и INSERT блоках.
|
||||
CREATE TEMP TABLE tmp_passengers_src ON COMMIT DROP AS
|
||||
SELECT
|
||||
d.passenger_id,
|
||||
d.passenger_name
|
||||
FROM (
|
||||
SELECT
|
||||
t.passenger_id,
|
||||
t.passenger_name,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY t.passenger_id
|
||||
ORDER BY t.event_ts DESC NULLS LAST, t._load_ts DESC, t.ticket_no DESC
|
||||
) AS rn
|
||||
FROM ods.tickets AS t
|
||||
WHERE t.passenger_id IS NOT NULL
|
||||
AND t.passenger_id <> ''
|
||||
AND t.passenger_name IS NOT NULL
|
||||
AND t.passenger_name <> ''
|
||||
) AS d
|
||||
WHERE d.rn = 1;
|
||||
|
||||
-- Statement 1: UPDATE существующих записей (если атрибуты изменились).
|
||||
UPDATE dds.dim_passengers AS d
|
||||
SET passenger_name = s.passenger_name,
|
||||
updated_at = now(),
|
||||
_load_id = '{{ run_id }}',
|
||||
_load_ts = now()
|
||||
FROM tmp_passengers_src AS s
|
||||
WHERE d.passenger_id = s.passenger_id
|
||||
AND d.passenger_name IS DISTINCT FROM s.passenger_name;
|
||||
|
||||
-- Statement 2: INSERT новых записей (MAX(sk) + ROW_NUMBER()).
|
||||
WITH max_sk AS (
|
||||
-- Учебный комментарий: Генерация SK через MAX() + ROW_NUMBER()
|
||||
-- Этот подход работает безопасно только потому, что Airflow запускает
|
||||
-- джобы загрузки для одной таблицы строго последовательно (concurrency=1).
|
||||
SELECT COALESCE(MAX(passenger_sk), 0) AS v
|
||||
FROM dds.dim_passengers
|
||||
)
|
||||
INSERT INTO dds.dim_passengers (
|
||||
passenger_sk,
|
||||
passenger_id,
|
||||
passenger_name,
|
||||
created_at,
|
||||
updated_at,
|
||||
_load_id,
|
||||
_load_ts
|
||||
)
|
||||
SELECT
|
||||
(SELECT v FROM max_sk) + ROW_NUMBER() OVER (ORDER BY s.passenger_id)::INTEGER,
|
||||
s.passenger_id,
|
||||
s.passenger_name,
|
||||
now(),
|
||||
now(),
|
||||
'{{ run_id }}',
|
||||
now()
|
||||
FROM tmp_passengers_src AS s
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM dds.dim_passengers AS d
|
||||
WHERE d.passenger_id = s.passenger_id
|
||||
);
|
||||
|
||||
ANALYZE dds.dim_passengers;
|
||||
-- TODO: реализуйте загрузку dds.dim_passengers (см. ТЗ в docs/assignment/analyst_spec.md)
|
||||
-- Паттерн: SCD1 UPSERT (UPDATE изменившихся + INSERT новых).
|
||||
-- Эталонную реализацию можно найти в ветке solution.
|
||||
SELECT 1;
|
||||
|
||||
+3
-157
@@ -1,157 +1,3 @@
|
||||
-- DQ для DDS dim_routes (SCD2).
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
v_row_count BIGINT;
|
||||
v_dup_sk BIGINT;
|
||||
v_dup_current BIGINT;
|
||||
v_overlap_count BIGINT;
|
||||
v_missing_count BIGINT;
|
||||
v_orphan_current BIGINT;
|
||||
v_null_count BIGINT;
|
||||
BEGIN
|
||||
-- Таблица не пуста.
|
||||
SELECT COUNT(*)
|
||||
INTO v_row_count
|
||||
FROM dds.dim_routes;
|
||||
|
||||
IF v_row_count = 0 THEN
|
||||
RAISE EXCEPTION 'DQ FAILED: dds.dim_routes пуста.';
|
||||
END IF;
|
||||
|
||||
-- Нет дублей по SK.
|
||||
SELECT COUNT(*) - COUNT(DISTINCT route_sk)
|
||||
INTO v_dup_sk
|
||||
FROM dds.dim_routes;
|
||||
|
||||
IF v_dup_sk <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_routes найдены дубликаты route_sk: %',
|
||||
v_dup_sk;
|
||||
END IF;
|
||||
|
||||
-- Корректность интервалов (valid_from <= valid_to для закрытых версий).
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_count
|
||||
FROM dds.dim_routes
|
||||
WHERE valid_to IS NOT NULL
|
||||
AND valid_from > valid_to;
|
||||
|
||||
IF v_null_count <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_routes найдены версии с valid_from > valid_to: %',
|
||||
v_null_count;
|
||||
END IF;
|
||||
|
||||
-- Нет перекрытий интервалов для одного route_bk.
|
||||
SELECT COUNT(*)
|
||||
INTO v_overlap_count
|
||||
FROM (
|
||||
SELECT 1
|
||||
FROM dds.dim_routes AS d1
|
||||
JOIN dds.dim_routes AS d2
|
||||
ON d1.route_bk = d2.route_bk
|
||||
AND d1.route_sk < d2.route_sk
|
||||
AND d1.valid_from < COALESCE(d2.valid_to, DATE '9999-12-31')
|
||||
AND d2.valid_from < COALESCE(d1.valid_to, DATE '9999-12-31')
|
||||
) AS overlap_rows;
|
||||
|
||||
IF v_overlap_count <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_routes найдены перекрытия SCD2-интервалов: %',
|
||||
v_overlap_count;
|
||||
END IF;
|
||||
|
||||
-- Не более одной текущей версии на route_bk.
|
||||
SELECT COUNT(*)
|
||||
INTO v_dup_current
|
||||
FROM (
|
||||
SELECT route_bk
|
||||
FROM dds.dim_routes
|
||||
WHERE valid_to IS NULL
|
||||
GROUP BY route_bk
|
||||
HAVING COUNT(*) > 1
|
||||
) AS d;
|
||||
|
||||
IF v_dup_current <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_routes найдены route_bk с > 1 текущей версией: %',
|
||||
v_dup_current;
|
||||
END IF;
|
||||
|
||||
-- Покрытие ODS: все route_no имеют хотя бы одну версию в DDS.
|
||||
SELECT COUNT(*)
|
||||
INTO v_missing_count
|
||||
FROM (SELECT DISTINCT route_no FROM ods.routes) AS o
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM dds.dim_routes AS d
|
||||
WHERE d.route_bk = o.route_no
|
||||
);
|
||||
|
||||
IF v_missing_count <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_routes отсутствуют маршруты из ODS: %',
|
||||
v_missing_count;
|
||||
END IF;
|
||||
|
||||
-- Current-срез DDS не содержит route_bk, которых нет в ODS.
|
||||
SELECT COUNT(*)
|
||||
INTO v_orphan_current
|
||||
FROM (
|
||||
SELECT DISTINCT route_bk
|
||||
FROM dds.dim_routes
|
||||
WHERE valid_to IS NULL
|
||||
) AS d
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ods.routes AS o
|
||||
WHERE o.route_no = d.route_bk
|
||||
);
|
||||
|
||||
IF v_orphan_current <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в current-срезе dds.dim_routes есть route_bk вне ODS: %',
|
||||
v_orphan_current;
|
||||
END IF;
|
||||
|
||||
-- Обязательные поля.
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_count
|
||||
FROM dds.dim_routes
|
||||
WHERE route_sk IS NULL
|
||||
OR route_bk IS NULL
|
||||
OR route_bk = ''
|
||||
OR departure_airport IS NULL
|
||||
OR departure_airport = ''
|
||||
OR arrival_airport IS NULL
|
||||
OR arrival_airport = ''
|
||||
OR airplane_code IS NULL
|
||||
OR airplane_code = ''
|
||||
OR departure_city IS NULL
|
||||
OR departure_city = ''
|
||||
OR arrival_city IS NULL
|
||||
OR arrival_city = ''
|
||||
OR airplane_model IS NULL
|
||||
OR airplane_model = ''
|
||||
OR total_seats IS NULL
|
||||
OR hashdiff IS NULL
|
||||
OR hashdiff = ''
|
||||
OR valid_from IS NULL
|
||||
OR created_at IS NULL
|
||||
OR updated_at IS NULL
|
||||
OR _load_id IS NULL
|
||||
OR _load_id = ''
|
||||
OR _load_ts IS NULL;
|
||||
|
||||
IF v_null_count <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в dds.dim_routes найдены NULL обязательные поля: %',
|
||||
v_null_count;
|
||||
END IF;
|
||||
|
||||
RAISE NOTICE
|
||||
'DQ PASSED: dds.dim_routes ок, строк=% (версий)',
|
||||
v_row_count;
|
||||
|
||||
END $$;
|
||||
-- TODO: реализуйте проверки качества данных для dds.dim_routes (SCD2)
|
||||
-- Эталонную реализацию можно найти в ветке solution.
|
||||
SELECT 1;
|
||||
|
||||
+4
-144
@@ -1,144 +1,4 @@
|
||||
-- Загрузка DDS dim_routes: SCD2 с hashdiff.
|
||||
|
||||
-- Учебный комментарий: Используем TEMP TABLE для подготовки дельты.
|
||||
-- Это избавляет от дублирования логики hashdiff в UPDATE и INSERT блоках.
|
||||
CREATE TEMP TABLE tmp_routes_src ON COMMIT DROP AS
|
||||
SELECT
|
||||
route_no,
|
||||
departure_airport,
|
||||
arrival_airport,
|
||||
airplane_code,
|
||||
days_of_week,
|
||||
departure_time,
|
||||
duration,
|
||||
md5(
|
||||
COALESCE(departure_airport, '') || '|' ||
|
||||
COALESCE(arrival_airport, '') || '|' ||
|
||||
COALESCE(airplane_code, '') || '|' ||
|
||||
COALESCE(days_of_week::TEXT, '') || '|' ||
|
||||
COALESCE(departure_time::TEXT, '') || '|' ||
|
||||
COALESCE(duration::TEXT, '')
|
||||
) AS hashdiff,
|
||||
ROW_NUMBER() OVER (PARTITION BY route_no ORDER BY validity DESC) AS rn
|
||||
FROM ods.routes;
|
||||
|
||||
-- Statement 1: Закрыть устаревшие версии (valid_to = текущая дата).
|
||||
UPDATE dds.dim_routes AS d
|
||||
SET valid_to = CURRENT_DATE,
|
||||
updated_at = now(),
|
||||
_load_id = '{{ run_id }}',
|
||||
_load_ts = now()
|
||||
FROM tmp_routes_src AS s
|
||||
WHERE s.rn = 1
|
||||
AND d.route_bk = s.route_no
|
||||
AND d.valid_to IS NULL
|
||||
AND d.hashdiff <> s.hashdiff;
|
||||
|
||||
-- Statement 1.1: Закрыть "исчезнувшие" маршруты.
|
||||
UPDATE dds.dim_routes AS d
|
||||
SET valid_to = CURRENT_DATE,
|
||||
updated_at = now(),
|
||||
_load_id = '{{ run_id }}',
|
||||
_load_ts = now()
|
||||
WHERE d.valid_to IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM tmp_routes_src AS s
|
||||
WHERE s.rn = 1
|
||||
AND s.route_no = d.route_bk
|
||||
);
|
||||
|
||||
-- Statement 2: Вставить новые версии (для изменённых и новых route_no).
|
||||
WITH max_sk AS (
|
||||
-- Учебный комментарий: Генерация SK через MAX() + ROW_NUMBER()
|
||||
-- Этот подход работает безопасно только потому, что Airflow запускает
|
||||
-- джобы загрузки для одной таблицы строго последовательно (concurrency=1).
|
||||
SELECT COALESCE(MAX(route_sk), 0) AS v
|
||||
FROM dds.dim_routes
|
||||
)
|
||||
INSERT INTO dds.dim_routes (
|
||||
route_sk,
|
||||
route_bk,
|
||||
departure_airport,
|
||||
arrival_airport,
|
||||
airplane_code,
|
||||
departure_city,
|
||||
arrival_city,
|
||||
airplane_model,
|
||||
total_seats,
|
||||
days_of_week,
|
||||
departure_time,
|
||||
duration,
|
||||
hashdiff,
|
||||
valid_from,
|
||||
valid_to,
|
||||
created_at,
|
||||
updated_at,
|
||||
_load_id,
|
||||
_load_ts
|
||||
)
|
||||
SELECT
|
||||
(SELECT v FROM max_sk) + ROW_NUMBER() OVER (ORDER BY s.route_no)::INTEGER,
|
||||
s.route_no,
|
||||
s.departure_airport,
|
||||
s.arrival_airport,
|
||||
s.airplane_code,
|
||||
dep.city,
|
||||
arr.city,
|
||||
air.model,
|
||||
air.total_seats,
|
||||
s.days_of_week,
|
||||
s.departure_time,
|
||||
s.duration,
|
||||
s.hashdiff,
|
||||
CASE
|
||||
WHEN EXISTS (
|
||||
SELECT 1
|
||||
FROM dds.dim_routes AS d2
|
||||
WHERE d2.route_bk = s.route_no
|
||||
) THEN CURRENT_DATE
|
||||
ELSE '1900-01-01'::DATE
|
||||
END AS valid_from,
|
||||
NULL,
|
||||
now(),
|
||||
now(),
|
||||
'{{ run_id }}',
|
||||
now()
|
||||
FROM tmp_routes_src AS s
|
||||
LEFT JOIN dds.dim_airports AS dep ON s.departure_airport = dep.airport_bk
|
||||
LEFT JOIN dds.dim_airports AS arr ON s.arrival_airport = arr.airport_bk
|
||||
LEFT JOIN dds.dim_airplanes AS air ON s.airplane_code = air.airplane_bk
|
||||
WHERE s.rn = 1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM dds.dim_routes AS d
|
||||
WHERE d.route_bk = s.route_no
|
||||
AND d.valid_to IS NULL
|
||||
AND d.hashdiff = s.hashdiff
|
||||
);
|
||||
|
||||
-- Фаза 3: Обновление денормализованных атрибутов (refresh).
|
||||
-- Нужна для SCD1-изменений в dim_airports/dim_airplanes (напр. переименование города).
|
||||
-- Обновляем все версии (и текущие, и исторические), т.к. измерения SCD1.
|
||||
-- При этом мы не перезаписываем _load_id и _load_ts, чтобы не размывать lineage версий.
|
||||
UPDATE dds.dim_routes AS d
|
||||
SET departure_city = dep.city,
|
||||
arrival_city = arr.city,
|
||||
airplane_model = air.model,
|
||||
total_seats = air.total_seats,
|
||||
updated_at = now()
|
||||
FROM dds.dim_airports AS dep,
|
||||
dds.dim_airports AS arr,
|
||||
dds.dim_airplanes AS air
|
||||
WHERE dep.airport_bk = d.departure_airport
|
||||
AND arr.airport_bk = d.arrival_airport
|
||||
AND air.airplane_bk = d.airplane_code
|
||||
AND (
|
||||
d.departure_city IS DISTINCT FROM dep.city
|
||||
OR d.arrival_city IS DISTINCT FROM arr.city
|
||||
OR d.airplane_model IS DISTINCT FROM air.model
|
||||
OR d.total_seats IS DISTINCT FROM air.total_seats
|
||||
);
|
||||
|
||||
ANALYZE dds.dim_routes;
|
||||
|
||||
-- TODO: реализуйте загрузку dds.dim_routes (см. ТЗ в docs/assignment/analyst_spec.md)
|
||||
-- Паттерн: SCD2 с hashdiff — самый нетривиальный паттерн в проекте.
|
||||
-- Эталонную реализацию можно найти в ветке solution.
|
||||
SELECT 1;
|
||||
|
||||
@@ -7,7 +7,8 @@ DECLARE
|
||||
v_dup_count BIGINT;
|
||||
v_null_passenger BIGINT;
|
||||
v_null_tariff BIGINT;
|
||||
v_null_route_related BIGINT;
|
||||
v_null_airport BIGINT;
|
||||
v_null_student BIGINT;
|
||||
v_null_calendar BIGINT;
|
||||
v_null_required BIGINT;
|
||||
BEGIN
|
||||
@@ -62,15 +63,17 @@ BEGIN
|
||||
v_dup_count;
|
||||
END IF;
|
||||
|
||||
-- Ссылочная целостность: passenger_sk.
|
||||
-- Учебный комментарий: passenger_sk будет NULL, пока вы не реализуете dim_passengers.
|
||||
-- После реализации: TRUNCATE dds.fact_flight_sales → перезагрузка → все SK заполнены.
|
||||
-- Полную версию DQ (с блокировкой) см. в ветке solution.
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_passenger
|
||||
FROM dds.fact_flight_sales
|
||||
WHERE passenger_sk IS NULL;
|
||||
|
||||
IF v_null_passenger <> 0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в fact_flight_sales строки без passenger_sk: %',
|
||||
RAISE NOTICE
|
||||
'DQ INFO: студенческий SK (passenger) NULL: %. После реализации dim_passengers: TRUNCATE fact → перезагрузка.',
|
||||
v_null_passenger;
|
||||
END IF;
|
||||
|
||||
@@ -86,27 +89,42 @@ BEGIN
|
||||
v_null_tariff;
|
||||
END IF;
|
||||
|
||||
-- Route-related FK: допустимо при аномалиях, фейлим если > 1%.
|
||||
-- departure_airport_sk и arrival_airport_sk заполняются через ods.routes (эталон).
|
||||
-- NULL здесь — аномалия данных (пропущен маршрут в ODS), а не отсутствие студенческого кода.
|
||||
-- Порог 1% — защита от единичных аномалий источника (аналогично calendar_sk).
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_route_related
|
||||
INTO v_null_airport
|
||||
FROM dds.fact_flight_sales
|
||||
WHERE route_sk IS NULL
|
||||
OR departure_airport_sk IS NULL
|
||||
OR arrival_airport_sk IS NULL
|
||||
OR airplane_sk IS NULL;
|
||||
WHERE departure_airport_sk IS NULL
|
||||
OR arrival_airport_sk IS NULL;
|
||||
|
||||
IF v_null_route_related > 0 THEN
|
||||
IF v_null_route_related * 100.0 / NULLIF(v_row_count, 0) > 1.0 THEN
|
||||
IF v_null_airport > 0 THEN
|
||||
IF v_null_airport * 100.0 / NULLIF(v_row_count, 0) > 1.0 THEN
|
||||
RAISE EXCEPTION
|
||||
'DQ FAILED: в fact_flight_sales слишком много строк с NULL в route-related FK: % (>1%%)',
|
||||
v_null_route_related;
|
||||
'DQ FAILED: NULL airport_sk: % (>1%%)',
|
||||
v_null_airport;
|
||||
ELSE
|
||||
RAISE NOTICE
|
||||
'DQ WARNING: в fact_flight_sales строк с NULL в route-related FK: % (<=1%%, допустимо)',
|
||||
v_null_route_related;
|
||||
'DQ WARNING: NULL airport_sk: % (<=1%%, допустимо)',
|
||||
v_null_airport;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- route_sk и airplane_sk будут NULL, пока вы не реализуете dim_routes.
|
||||
-- Не блокируем pipeline.
|
||||
-- Полную версию DQ (с блокировкой) см. в ветке solution.
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_student
|
||||
FROM dds.fact_flight_sales
|
||||
WHERE route_sk IS NULL
|
||||
OR airplane_sk IS NULL;
|
||||
|
||||
IF v_null_student > 0 THEN
|
||||
RAISE NOTICE
|
||||
'DQ INFO: студенческие SK (route/airplane) NULL: %. После реализации dim_routes: TRUNCATE fact → перезагрузка.',
|
||||
v_null_student;
|
||||
END IF;
|
||||
|
||||
-- Calendar: допустимо если scheduled_departure IS NULL, фейлим если > 1%.
|
||||
SELECT COUNT(*)
|
||||
INTO v_null_calendar
|
||||
|
||||
Reference in New Issue
Block a user