docs(labs): добавлены эталон и протокол лабораторной PXF
This commit is contained in:
1 parent
7bf61d8f74
commit
d105cb75df
9 files changed
+785
-10
No files matched your search
@@ -0,0 +1,80 @@
|
||||
"""Проверка должна отвергать ошибки, которые не видны по количеству строк."""
|
||||
|
||||
import importlib.util
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"check_airports_snapshot",
|
||||
Path(__file__).resolve().parents[1] / "scripts/check_airports_snapshot.py",
|
||||
)
|
||||
checker = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(checker)
|
||||
|
||||
SOURCE = [
|
||||
{
|
||||
"airport_code": "AAA",
|
||||
"airport_name": '{"en": "Airport", "ru": "Аэропорт"}',
|
||||
"city": '{"ru": "Город"}',
|
||||
"country": '{"ru": "Страна"}',
|
||||
"coordinates": "(1,2)",
|
||||
"timezone": "Europe/Moscow",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def snapshot():
|
||||
"""Снимок с одним ключом и обязательными метками загрузки."""
|
||||
rows = deepcopy(SOURCE)
|
||||
rows[0].update(
|
||||
_load_id="B", _load_ts="2026-09-22 12:00:00", event_ts="2026-09-22 12:00:00"
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def test_accepts_complete_snapshot_with_json_key_order_difference():
|
||||
rows = snapshot()
|
||||
rows[0]["airport_name"] = '{"ru":"Аэропорт","en":"Airport"}'
|
||||
assert checker.compare_snapshot(SOURCE, rows, "B", "stg") == []
|
||||
|
||||
|
||||
def test_accepts_different_row_order_from_segments():
|
||||
"""Порядок выдачи строк Greenplum не меняет содержимое снимка."""
|
||||
expected = deepcopy(SOURCE) + deepcopy(SOURCE)
|
||||
expected[1]["airport_code"] = "BBB"
|
||||
rows = snapshot() + snapshot()
|
||||
rows[1]["airport_code"] = "BBB"
|
||||
assert checker.compare_snapshot(expected, rows[::-1], "B", "stg") == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value"),
|
||||
[
|
||||
("airport_code", "BBB"),
|
||||
("airport_name", '{"ru":"Старое имя","en":"Airport"}'),
|
||||
("country", '{"ru":"Другая страна"}'),
|
||||
("coordinates", "(2,1)"),
|
||||
("_load_id", "A"),
|
||||
("event_ts", None),
|
||||
],
|
||||
)
|
||||
def test_rejects_wrong_values_despite_equal_row_count(field, value):
|
||||
rows = snapshot()
|
||||
rows[0][field] = value
|
||||
assert checker.compare_snapshot(SOURCE, rows, "B", "stg")
|
||||
|
||||
|
||||
def test_rejects_duplicate_and_empty_snapshot():
|
||||
rows = snapshot()
|
||||
assert checker.compare_snapshot(SOURCE, rows + rows, "B", "stg")
|
||||
assert checker.compare_snapshot(SOURCE, [], "B", "stg")
|
||||
|
||||
|
||||
def test_ods_requires_both_transformation_and_selected_batch():
|
||||
rows = snapshot()
|
||||
rows[0].update(airport_name="Аэропорт", city="Город", country="Страна")
|
||||
assert checker.compare_snapshot(SOURCE, rows, "B", "ods") == []
|
||||
rows[0]["_load_id"] = "A"
|
||||
assert checker.compare_snapshot(SOURCE, rows, "B", "ods")
|
||||
@@ -66,6 +66,22 @@ def test_bookings_stg_ddl_dag_structure():
|
||||
_assert_reachable(dag, "apply_stg_bookings_ddl", task_id)
|
||||
|
||||
|
||||
def test_lab_pxf_airports_uses_snapshot_sql_and_reference_dependencies():
|
||||
"""Опыт читает четыре справочника; генератор и транзакции в него не входят."""
|
||||
dag = _load_dag("airflow.dags.lab_pxf_airports")
|
||||
assert len(dag.tasks) == 8
|
||||
for entity in ("airports", "airplanes", "routes", "seats"):
|
||||
load = dag.get_task(f"load_{entity}_to_stg")
|
||||
dq = dag.get_task(f"check_{entity}_dq")
|
||||
assert load.sql == f"stg/{entity}_load.sql"
|
||||
assert dq.sql == f"stg/{entity}_dq.sql"
|
||||
assert load.postgres_conn_id == dq.postgres_conn_id == "greenplum_conn"
|
||||
_assert_direct_edge(dag, load.task_id, dq.task_id)
|
||||
_assert_direct_edge(dag, "check_airports_dq", "load_routes_to_stg")
|
||||
_assert_direct_edge(dag, "check_airplanes_dq", "load_routes_to_stg")
|
||||
_assert_direct_edge(dag, "check_airplanes_dq", "load_seats_to_stg")
|
||||
|
||||
|
||||
def test_bookings_to_gp_stage_dag_structure():
|
||||
"""Проверка структуры DAG bookings_to_gp_stage."""
|
||||
dag = _load_dag("airflow.dags.bookings_to_gp_stage")
|
||||
|
||||
Reference in new issue
Block a user