docs(labs): добавлены эталон и протокол лабораторной PXF

This commit is contained in:
ddadmin committed 2026-09-22 11:26:27 -04:00
1 parent 7bf61d8f74
commit d105cb75df
9 files changed
+785 -10

No files matched your search

+80
View File
@@ -0,0 +1,80 @@
"""Проверка должна отвергать ошибки, которые не видны по количеству строк."""
import importlib.util
from copy import deepcopy
from pathlib import Path
import pytest
spec = importlib.util.spec_from_file_location(
"check_airports_snapshot",
Path(__file__).resolve().parents[1] / "scripts/check_airports_snapshot.py",
)
checker = importlib.util.module_from_spec(spec)
spec.loader.exec_module(checker)
SOURCE = [
{
"airport_code": "AAA",
"airport_name": '{"en": "Airport", "ru": "Аэропорт"}',
"city": '{"ru": "Город"}',
"country": '{"ru": "Страна"}',
"coordinates": "(1,2)",
"timezone": "Europe/Moscow",
}
]
def snapshot():
"""Снимок с одним ключом и обязательными метками загрузки."""
rows = deepcopy(SOURCE)
rows[0].update(
_load_id="B", _load_ts="2026-09-22 12:00:00", event_ts="2026-09-22 12:00:00"
)
return rows
def test_accepts_complete_snapshot_with_json_key_order_difference():
rows = snapshot()
rows[0]["airport_name"] = '{"ru":"Аэропорт","en":"Airport"}'
assert checker.compare_snapshot(SOURCE, rows, "B", "stg") == []
def test_accepts_different_row_order_from_segments():
"""Порядок выдачи строк Greenplum не меняет содержимое снимка."""
expected = deepcopy(SOURCE) + deepcopy(SOURCE)
expected[1]["airport_code"] = "BBB"
rows = snapshot() + snapshot()
rows[1]["airport_code"] = "BBB"
assert checker.compare_snapshot(expected, rows[::-1], "B", "stg") == []
@pytest.mark.parametrize(
("field", "value"),
[
("airport_code", "BBB"),
("airport_name", '{"ru":"Старое имя","en":"Airport"}'),
("country", '{"ru":"Другая страна"}'),
("coordinates", "(2,1)"),
("_load_id", "A"),
("event_ts", None),
],
)
def test_rejects_wrong_values_despite_equal_row_count(field, value):
rows = snapshot()
rows[0][field] = value
assert checker.compare_snapshot(SOURCE, rows, "B", "stg")
def test_rejects_duplicate_and_empty_snapshot():
rows = snapshot()
assert checker.compare_snapshot(SOURCE, rows + rows, "B", "stg")
assert checker.compare_snapshot(SOURCE, [], "B", "stg")
def test_ods_requires_both_transformation_and_selected_batch():
rows = snapshot()
rows[0].update(airport_name="Аэропорт", city="Город", country="Страна")
assert checker.compare_snapshot(SOURCE, rows, "B", "ods") == []
rows[0]["_load_id"] = "A"
assert checker.compare_snapshot(SOURCE, rows, "B", "ods")
+16
View File
@@ -66,6 +66,22 @@ def test_bookings_stg_ddl_dag_structure():
_assert_reachable(dag, "apply_stg_bookings_ddl", task_id)
def test_lab_pxf_airports_uses_snapshot_sql_and_reference_dependencies():
"""Опыт читает четыре справочника; генератор и транзакции в него не входят."""
dag = _load_dag("airflow.dags.lab_pxf_airports")
assert len(dag.tasks) == 8
for entity in ("airports", "airplanes", "routes", "seats"):
load = dag.get_task(f"load_{entity}_to_stg")
dq = dag.get_task(f"check_{entity}_dq")
assert load.sql == f"stg/{entity}_load.sql"
assert dq.sql == f"stg/{entity}_dq.sql"
assert load.postgres_conn_id == dq.postgres_conn_id == "greenplum_conn"
_assert_direct_edge(dag, load.task_id, dq.task_id)
_assert_direct_edge(dag, "check_airports_dq", "load_routes_to_stg")
_assert_direct_edge(dag, "check_airplanes_dq", "load_routes_to_stg")
_assert_direct_edge(dag, "check_airplanes_dq", "load_seats_to_stg")
def test_bookings_to_gp_stage_dag_structure():
"""Проверка структуры DAG bookings_to_gp_stage."""
dag = _load_dag("airflow.dags.bookings_to_gp_stage")