refactor(csv): удален легаси CSV-пайплайн и связанные с ним файлы
- Зачем: - Пример базовой загрузки CSV перенесен в отдельный репозиторий `airflow-manual` для разделения учебных треков. - Что: - удалены DAG-файлы `csv_to_greenplum` и вспомогательные скрипты `helpers/greenplum.py`, `orders_ddl.sql`. - из `docker-compose.yml` и `.env.example` удалены переменные и тома (`airflow_data`), необходимые для CSV. - очищена документация (`README.md`, `TESTING.md`, `educational-tasks.md`) и тесты (`test_dags_smoke.py`, `conftest.py`). - отмечен выполненным 'Этап 1' в `TODO.md`. - Проверка: - `make test` проходит успешно (smoke-тесты оставшихся DAG-ов не затронуты).
This commit is contained in:
@@ -55,14 +55,3 @@ def _ensure_stub_module(full_name: str) -> ModuleType:
|
||||
module = sys.modules[path]
|
||||
assert isinstance(module, ModuleType)
|
||||
return module
|
||||
|
||||
|
||||
def patch_postgres_hook(monkeypatch, hook_cls: Type) -> None:
|
||||
"""
|
||||
Patch PostgresHook so that helpers.greenplum can be exercised without real Airflow.
|
||||
"""
|
||||
try:
|
||||
module = importlib.import_module("airflow.providers.postgres.hooks.postgres")
|
||||
except ModuleNotFoundError:
|
||||
module = _ensure_stub_module("airflow.providers.postgres.hooks.postgres")
|
||||
monkeypatch.setattr(module, "PostgresHook", hook_cls, raising=False)
|
||||
|
||||
@@ -41,53 +41,6 @@ def _assert_reachable(dag, upstream_task_id: str, downstream_task_id: str) -> No
|
||||
), f"Expected {downstream_task_id} to be downstream of {upstream_task_id}"
|
||||
|
||||
|
||||
def test_csv_to_greenplum_dag_structure():
|
||||
dag = _load_dag("airflow.dags.csv_to_greenplum")
|
||||
|
||||
# tasks
|
||||
expected_tasks = {
|
||||
"create_orders_table",
|
||||
"generate_csv",
|
||||
"preview_csv",
|
||||
"load_csv_to_greenplum",
|
||||
}
|
||||
assert expected_tasks.issubset(dag.task_dict.keys())
|
||||
|
||||
# linear dependencies
|
||||
t1 = dag.get_task("create_orders_table")
|
||||
t2 = dag.get_task("generate_csv")
|
||||
t3 = dag.get_task("preview_csv")
|
||||
t4 = dag.get_task("load_csv_to_greenplum")
|
||||
|
||||
assert t2 in t1.get_direct_relatives(upstream=False)
|
||||
assert t3 in t2.get_direct_relatives(upstream=False)
|
||||
assert t4 in t3.get_direct_relatives(upstream=False)
|
||||
|
||||
|
||||
def test_csv_to_greenplum_dq_dag_structure():
|
||||
dag = _load_dag("airflow.dags.csv_to_greenplum_dq")
|
||||
|
||||
expected_tasks = {
|
||||
"check_orders_table_exists",
|
||||
"check_orders_schema",
|
||||
"check_orders_has_rows",
|
||||
"check_order_duplicates",
|
||||
"data_quality_summary",
|
||||
}
|
||||
assert expected_tasks.issubset(dag.task_dict.keys())
|
||||
|
||||
e = dag.get_task("check_orders_table_exists")
|
||||
s = dag.get_task("check_orders_schema")
|
||||
h = dag.get_task("check_orders_has_rows")
|
||||
d = dag.get_task("check_order_duplicates")
|
||||
q = dag.get_task("data_quality_summary")
|
||||
|
||||
assert s in e.get_direct_relatives(upstream=False)
|
||||
assert h in s.get_direct_relatives(upstream=False)
|
||||
assert d in h.get_direct_relatives(upstream=False)
|
||||
assert q in d.get_direct_relatives(upstream=False)
|
||||
|
||||
|
||||
def test_bookings_stg_ddl_dag_structure():
|
||||
"""Проверка структуры DAG bookings_stg_ddl."""
|
||||
dag = _load_dag("airflow.dags.bookings_stg_ddl")
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
import airflow.dags.helpers.greenplum as greenplum
|
||||
from tests.conftest import patch_postgres_hook
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeCursor:
|
||||
fetchone_value: Any = None
|
||||
fetchall_value: Sequence[Any] | None = None
|
||||
rowcount: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.queries: List[Any] = []
|
||||
|
||||
def execute(self, query: str, params: Any | None = None) -> None:
|
||||
self.queries.append((query, params))
|
||||
|
||||
def fetchone(self) -> Any:
|
||||
return self.fetchone_value
|
||||
|
||||
def fetchall(self) -> Sequence[Any] | None:
|
||||
return self.fetchall_value
|
||||
|
||||
def __enter__(self) -> FakeCursor:
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class FakeConn:
|
||||
def __init__(self, cursors: Sequence[FakeCursor]) -> None:
|
||||
self._cursors = list(cursors)
|
||||
self._index = 0
|
||||
self.commits = 0
|
||||
|
||||
def cursor(self) -> FakeCursor:
|
||||
cursor = self._cursors[self._index]
|
||||
self._index += 1
|
||||
return cursor
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
|
||||
def test_get_gp_conn_uses_airflow_hook(monkeypatch) -> None:
|
||||
class FakeHook:
|
||||
def __init__(self, postgres_conn_id: str) -> None:
|
||||
self.postgres_conn_id = postgres_conn_id
|
||||
|
||||
def get_conn(self) -> str:
|
||||
return "hook_connection"
|
||||
|
||||
patch_postgres_hook(monkeypatch, FakeHook)
|
||||
monkeypatch.setattr(greenplum, "GP_CONN_ID", "demo_conn", raising=False)
|
||||
monkeypatch.setattr(greenplum, "GP_USE_AIRFLOW_CONN", True, raising=False)
|
||||
|
||||
conn = greenplum.get_gp_conn()
|
||||
|
||||
assert conn == "hook_connection"
|
||||
|
||||
|
||||
def test_get_gp_conn_fallback_to_psycopg(monkeypatch) -> None:
|
||||
class BrokenHook:
|
||||
def __init__(self, postgres_conn_id: str) -> None:
|
||||
self.postgres_conn_id = postgres_conn_id
|
||||
|
||||
def get_conn(self):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
patch_postgres_hook(monkeypatch, BrokenHook)
|
||||
monkeypatch.setattr(greenplum, "GP_USE_AIRFLOW_CONN", True, raising=False)
|
||||
monkeypatch.setattr(greenplum, "GP_CONN_ID", "demo_conn", raising=False)
|
||||
monkeypatch.setenv("GP_DB", "demo_db")
|
||||
monkeypatch.setenv("GP_USER", "demo_user")
|
||||
monkeypatch.setenv("GP_PASSWORD", "secret")
|
||||
monkeypatch.setenv("GP_HOST", "greenplum-host")
|
||||
monkeypatch.setenv("GP_PORT", "5434")
|
||||
|
||||
captured_kwargs = {}
|
||||
|
||||
def fake_connect(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return "psycopg_connection"
|
||||
|
||||
monkeypatch.setattr(greenplum.psycopg2, "connect", fake_connect)
|
||||
|
||||
conn = greenplum.get_gp_conn()
|
||||
|
||||
assert conn == "psycopg_connection"
|
||||
assert captured_kwargs == {
|
||||
"dbname": "demo_db",
|
||||
"user": "demo_user",
|
||||
"password": "secret",
|
||||
"host": "greenplum-host",
|
||||
"port": 5434,
|
||||
}
|
||||
|
||||
|
||||
def test_get_gp_conn_without_airflow(monkeypatch) -> None:
|
||||
monkeypatch.setattr(greenplum, "GP_USE_AIRFLOW_CONN", False, raising=False)
|
||||
monkeypatch.setenv("GP_DB", "demo_db")
|
||||
monkeypatch.setenv("GP_USER", "demo_user")
|
||||
monkeypatch.setenv("GP_PASSWORD", "secret")
|
||||
monkeypatch.setenv("GP_HOST", "greenplum-host")
|
||||
monkeypatch.setenv("GP_PORT", "5435")
|
||||
|
||||
captured_kwargs = {}
|
||||
|
||||
def fake_connect(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return "direct_psycopg"
|
||||
|
||||
monkeypatch.setattr(greenplum.psycopg2, "connect", fake_connect)
|
||||
|
||||
conn = greenplum.get_gp_conn()
|
||||
|
||||
assert conn == "direct_psycopg"
|
||||
assert captured_kwargs["port"] == 5435
|
||||
|
||||
|
||||
def test_assert_orders_table_exists_ok() -> None:
|
||||
conn = FakeConn([FakeCursor(fetchone_value=(1,))])
|
||||
|
||||
greenplum.assert_orders_table_exists(conn)
|
||||
|
||||
|
||||
def test_assert_orders_table_exists_missing() -> None:
|
||||
conn = FakeConn([FakeCursor(fetchone_value=None)])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
greenplum.assert_orders_table_exists(conn)
|
||||
|
||||
|
||||
def test_assert_orders_schema_ok() -> None:
|
||||
expected = list(greenplum.EXPECTED_ORDERS_SCHEMA)
|
||||
conn = FakeConn([FakeCursor(fetchall_value=expected)])
|
||||
|
||||
greenplum.assert_orders_schema(conn)
|
||||
|
||||
|
||||
def test_assert_orders_schema_mismatch() -> None:
|
||||
conn = FakeConn([FakeCursor(fetchall_value=[("order_id", "bigint")])])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
greenplum.assert_orders_schema(conn)
|
||||
|
||||
|
||||
def test_assert_orders_have_rows_ok() -> None:
|
||||
conn = FakeConn([FakeCursor(fetchone_value=(5,))])
|
||||
|
||||
greenplum.assert_orders_have_rows(conn)
|
||||
|
||||
|
||||
def test_assert_orders_have_rows_empty() -> None:
|
||||
conn = FakeConn([FakeCursor(fetchone_value=(0,))])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
greenplum.assert_orders_have_rows(conn)
|
||||
|
||||
|
||||
def test_assert_orders_no_duplicates_ok() -> None:
|
||||
conn = FakeConn([FakeCursor(fetchone_value=(0,))])
|
||||
|
||||
greenplum.assert_orders_no_duplicates(conn)
|
||||
|
||||
|
||||
def test_assert_orders_no_duplicates_detected() -> None:
|
||||
conn = FakeConn([FakeCursor(fetchone_value=(3,))])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
greenplum.assert_orders_no_duplicates(conn)
|
||||
Reference in New Issue
Block a user