feat(ods): реализован ODS слой и DAG загрузки из STG
- Зачем: - подготовлена учебная реализация ODS слоя с типизацией, UPSERT и DQ, чтобы продолжить работу от STG к DDS/DM. - Что: - добавлены SQL-скрипты `sql/ods/*_ddl.sql`, `sql/ods/*_load.sql`, `sql/ods/*_dq.sql` для 9 сущностей bookings. - добавлены DAG `bookings_ods_ddl` и `bookings_to_gp_ods`, а также smoke-тесты для новых графов. - ODS DDL интегрирован в `sql/ddl_gp.sql`; документация и план обновлены под единый запуск через `make ddl-gp`. - Проверка: - `make test`. - `make ddl-gp`.
This commit is contained in:
@@ -199,3 +199,101 @@ def test_bookings_to_gp_stage_dag_structure():
|
||||
assert airports not in airplanes.get_flat_relatives(
|
||||
upstream=False
|
||||
), "airplanes не должен быть upstream для airports"
|
||||
|
||||
|
||||
def test_bookings_ods_ddl_dag_structure():
|
||||
"""Проверка структуры DAG bookings_ods_ddl."""
|
||||
dag = _load_dag("airflow.dags.bookings_ods_ddl")
|
||||
|
||||
expected_tasks = {
|
||||
"apply_ods_airports_ddl",
|
||||
"apply_ods_airplanes_ddl",
|
||||
"apply_ods_routes_ddl",
|
||||
"apply_ods_seats_ddl",
|
||||
"apply_ods_bookings_ddl",
|
||||
"apply_ods_tickets_ddl",
|
||||
"apply_ods_flights_ddl",
|
||||
"apply_ods_segments_ddl",
|
||||
"apply_ods_boarding_passes_ddl",
|
||||
}
|
||||
assert expected_tasks.issubset(dag.task_dict.keys())
|
||||
|
||||
_assert_reachable(dag, "apply_ods_airports_ddl", "apply_ods_airplanes_ddl")
|
||||
|
||||
for task_id in expected_tasks - {"apply_ods_airports_ddl"}:
|
||||
_assert_reachable(dag, "apply_ods_airports_ddl", task_id)
|
||||
|
||||
|
||||
def test_bookings_to_gp_ods_dag_structure():
|
||||
"""Проверка структуры DAG bookings_to_gp_ods."""
|
||||
dag = _load_dag("airflow.dags.bookings_to_gp_ods")
|
||||
|
||||
expected_tasks = {
|
||||
"resolve_stg_batch_id",
|
||||
"load_ods_bookings",
|
||||
"dq_ods_bookings",
|
||||
"load_ods_tickets",
|
||||
"dq_ods_tickets",
|
||||
"load_ods_airports",
|
||||
"dq_ods_airports",
|
||||
"load_ods_airplanes",
|
||||
"dq_ods_airplanes",
|
||||
"load_ods_routes",
|
||||
"dq_ods_routes",
|
||||
"load_ods_seats",
|
||||
"dq_ods_seats",
|
||||
"load_ods_flights",
|
||||
"dq_ods_flights",
|
||||
"load_ods_segments",
|
||||
"dq_ods_segments",
|
||||
"load_ods_boarding_passes",
|
||||
"dq_ods_boarding_passes",
|
||||
"finish_ods_summary",
|
||||
}
|
||||
assert expected_tasks.issubset(dag.task_dict.keys())
|
||||
|
||||
# Базовая цепочка транзакций.
|
||||
_assert_reachable(dag, "resolve_stg_batch_id", "load_ods_bookings")
|
||||
_assert_reachable(dag, "load_ods_bookings", "dq_ods_bookings")
|
||||
_assert_reachable(dag, "dq_ods_bookings", "load_ods_tickets")
|
||||
_assert_reachable(dag, "load_ods_tickets", "dq_ods_tickets")
|
||||
|
||||
# Инвариант "load -> dq" для каждой таблицы.
|
||||
load_to_dq = [
|
||||
("load_ods_bookings", "dq_ods_bookings"),
|
||||
("load_ods_tickets", "dq_ods_tickets"),
|
||||
("load_ods_airports", "dq_ods_airports"),
|
||||
("load_ods_airplanes", "dq_ods_airplanes"),
|
||||
("load_ods_routes", "dq_ods_routes"),
|
||||
("load_ods_seats", "dq_ods_seats"),
|
||||
("load_ods_flights", "dq_ods_flights"),
|
||||
("load_ods_segments", "dq_ods_segments"),
|
||||
("load_ods_boarding_passes", "dq_ods_boarding_passes"),
|
||||
]
|
||||
for load_task_id, dq_task_id in load_to_dq:
|
||||
_assert_direct_edge(dag, load_task_id, dq_task_id)
|
||||
|
||||
# Справочники стартуют параллельно и не зависят друг от друга.
|
||||
_assert_reachable(dag, "resolve_stg_batch_id", "load_ods_airports")
|
||||
_assert_reachable(dag, "resolve_stg_batch_id", "load_ods_airplanes")
|
||||
airports = dag.get_task("load_ods_airports")
|
||||
airplanes = dag.get_task("load_ods_airplanes")
|
||||
assert airplanes not in airports.get_flat_relatives(
|
||||
upstream=False
|
||||
), "airports не должен быть upstream для airplanes"
|
||||
assert airports not in airplanes.get_flat_relatives(
|
||||
upstream=False
|
||||
), "airplanes не должен быть upstream для airports"
|
||||
|
||||
# Барьеры по данным.
|
||||
_assert_reachable(dag, "dq_ods_airports", "dq_ods_routes")
|
||||
_assert_reachable(dag, "dq_ods_airplanes", "dq_ods_routes")
|
||||
_assert_reachable(dag, "dq_ods_airplanes", "dq_ods_seats")
|
||||
_assert_reachable(dag, "dq_ods_routes", "dq_ods_flights")
|
||||
_assert_reachable(dag, "dq_ods_flights", "dq_ods_segments")
|
||||
_assert_reachable(dag, "dq_ods_tickets", "dq_ods_segments")
|
||||
_assert_reachable(dag, "dq_ods_segments", "dq_ods_boarding_passes")
|
||||
|
||||
# Финальная сводка должна ждать обе ветки.
|
||||
_assert_reachable(dag, "dq_ods_boarding_passes", "finish_ods_summary")
|
||||
_assert_reachable(dag, "dq_ods_seats", "finish_ods_summary")
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
RUN_ODS_INTEGRATION = os.getenv("RUN_ODS_INTEGRATION") == "1"
|
||||
BATCH_TOKEN = '{{ ti.xcom_pull(task_ids="resolve_stg_batch_id") }}'
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not RUN_ODS_INTEGRATION,
|
||||
reason="Set RUN_ODS_INTEGRATION=1 to run ODS integration tests",
|
||||
)
|
||||
|
||||
|
||||
def _psql(sql: str) -> str:
|
||||
"""Выполняет SQL в Greenplum-контейнере и возвращает stdout psql."""
|
||||
docker_bin = os.getenv("DOCKER_BIN", "docker")
|
||||
cmd = [
|
||||
docker_bin,
|
||||
"compose",
|
||||
"-f",
|
||||
"docker-compose.yml",
|
||||
"exec",
|
||||
"-T",
|
||||
"greenplum",
|
||||
"bash",
|
||||
"-lc",
|
||||
"su - gpadmin -c '/usr/local/greenplum-db/bin/psql -v ON_ERROR_STOP=1 -d gp_dwh -At -f -'",
|
||||
]
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
cwd=PROJECT_ROOT,
|
||||
input=sql,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _render_airports_sql(
|
||||
path: str, batch_id: str, stg_table: str, ods_table: str
|
||||
) -> str:
|
||||
sql = (PROJECT_ROOT / path).read_text(encoding="utf-8")
|
||||
sql = sql.replace(BATCH_TOKEN, batch_id)
|
||||
sql = sql.replace("stg.airports", stg_table)
|
||||
sql = sql.replace("ods.airports", ods_table)
|
||||
return sql
|
||||
|
||||
|
||||
def test_snapshot_airports_contract_upsert_delete_and_dq() -> None:
|
||||
"""
|
||||
Интеграционный тест контракта snapshot-таблицы:
|
||||
- UPSERT обновляет и вставляет;
|
||||
- DELETE синхронизирует current state по выбранному батчу;
|
||||
- DQ проходит на корректном состоянии.
|
||||
"""
|
||||
stg_table = "public.it_stg_airports_ods"
|
||||
ods_table = "public.it_ods_airports_ods"
|
||||
|
||||
setup_sql = f"""
|
||||
DROP TABLE IF EXISTS {stg_table};
|
||||
DROP TABLE IF EXISTS {ods_table};
|
||||
|
||||
CREATE TABLE {stg_table} (
|
||||
airport_code TEXT,
|
||||
airport_name TEXT,
|
||||
city TEXT,
|
||||
country TEXT,
|
||||
coordinates TEXT,
|
||||
timezone TEXT,
|
||||
src_created_at_ts TIMESTAMP,
|
||||
load_dttm TIMESTAMP,
|
||||
batch_id TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE {ods_table} (
|
||||
airport_code TEXT NOT NULL,
|
||||
airport_name TEXT NOT NULL,
|
||||
city TEXT NOT NULL,
|
||||
country TEXT NOT NULL,
|
||||
coordinates TEXT,
|
||||
timezone TEXT NOT NULL,
|
||||
_load_id TEXT NOT NULL,
|
||||
_load_ts TIMESTAMP NOT NULL DEFAULT now()
|
||||
);
|
||||
"""
|
||||
_psql(setup_sql)
|
||||
|
||||
try:
|
||||
_psql(
|
||||
f"""
|
||||
INSERT INTO {stg_table} (
|
||||
airport_code, airport_name, city, country, coordinates, timezone,
|
||||
src_created_at_ts, load_dttm, batch_id
|
||||
) VALUES
|
||||
('AAA', 'Airport A', 'City A', 'Country A', '(0,0)', 'UTC', now(), now(), 'batch_1'),
|
||||
('BBB', 'Airport B', 'City B', 'Country B', '(1,1)', 'UTC', now(), now(), 'batch_1');
|
||||
"""
|
||||
)
|
||||
|
||||
_psql(
|
||||
_render_airports_sql(
|
||||
"sql/ods/airports_load.sql", "batch_1", stg_table, ods_table
|
||||
)
|
||||
)
|
||||
|
||||
codes_batch_1 = _psql(
|
||||
f"SELECT COALESCE(string_agg(airport_code, ',' ORDER BY airport_code), '') FROM {ods_table};"
|
||||
).strip()
|
||||
assert codes_batch_1 == "AAA,BBB"
|
||||
|
||||
_psql(
|
||||
f"""
|
||||
INSERT INTO {stg_table} (
|
||||
airport_code, airport_name, city, country, coordinates, timezone,
|
||||
src_created_at_ts, load_dttm, batch_id
|
||||
) VALUES
|
||||
('AAA', 'Airport A v2', 'City A', 'Country A', '(0,0)', 'UTC', now(), now(), 'batch_2'),
|
||||
('CCC', 'Airport C', 'City C', 'Country C', '(2,2)', 'UTC', now(), now(), 'batch_2');
|
||||
"""
|
||||
)
|
||||
|
||||
_psql(
|
||||
_render_airports_sql(
|
||||
"sql/ods/airports_load.sql", "batch_2", stg_table, ods_table
|
||||
)
|
||||
)
|
||||
|
||||
codes_batch_2 = _psql(
|
||||
f"SELECT COALESCE(string_agg(airport_code, ',' ORDER BY airport_code), '') FROM {ods_table};"
|
||||
).strip()
|
||||
assert codes_batch_2 == "AAA,CCC"
|
||||
|
||||
airport_a_name = _psql(
|
||||
f"SELECT airport_name FROM {ods_table} WHERE airport_code = 'AAA';"
|
||||
).strip()
|
||||
assert airport_a_name == "Airport A v2"
|
||||
|
||||
_psql(
|
||||
_render_airports_sql(
|
||||
"sql/ods/airports_dq.sql", "batch_2", stg_table, ods_table
|
||||
)
|
||||
)
|
||||
finally:
|
||||
_psql(f"DROP TABLE IF EXISTS {stg_table}; DROP TABLE IF EXISTS {ods_table};")
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
SNAPSHOT_ENTITIES = ("airports", "airplanes", "routes", "seats")
|
||||
|
||||
|
||||
def _read(path: str) -> str:
|
||||
return (PROJECT_ROOT / path).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_snapshot_load_scripts_sync_deleted_keys() -> None:
|
||||
"""Snapshot-таблицы в ODS должны удалять ключи, отсутствующие в текущем батче."""
|
||||
for entity in SNAPSHOT_ENTITIES:
|
||||
sql = _read(f"sql/ods/{entity}_load.sql")
|
||||
assert f"DELETE FROM ods.{entity} AS o" in sql
|
||||
assert "WITH src_keys AS (" in sql
|
||||
assert "WHERE NOT EXISTS (" in sql
|
||||
|
||||
|
||||
def test_snapshot_dq_checks_extra_keys() -> None:
|
||||
"""DQ snapshot-таблиц должен ловить лишние ключи в ODS относительно текущего батча STG."""
|
||||
for entity in SNAPSHOT_ENTITIES:
|
||||
sql = _read(f"sql/ods/{entity}_dq.sql")
|
||||
assert "v_extra_keys_count" in sql
|
||||
assert "найдены лишние ключи" in sql
|
||||
|
||||
|
||||
def test_ods_batch_resolver_uses_consistent_snapshot_batches() -> None:
|
||||
"""Резолвер батча должен искать batch_id, общий для всех snapshot-таблиц STG."""
|
||||
dag_code = _read("airflow/dags/bookings_to_gp_ods.py")
|
||||
|
||||
for table_name in ("stg.airports", "stg.airplanes", "stg.routes", "stg.seats"):
|
||||
assert table_name in dag_code
|
||||
|
||||
assert "INTERSECT" in dag_code
|
||||
assert "candidate_batches" in dag_code
|
||||
Reference in New Issue
Block a user