Окружение для разработчика

airflow-greenplum/.gitignore

@@ -6,4 +6,5 @@
 __pycache__/
 */__pycache__/
 *.pyc
+.venv/
 data/

airflow-greenplum/.python-version

@@ -0,0 +1 @@
+3.11

airflow-greenplum/AGENTS.md

@@ -17,6 +17,13 @@
 - `make down` — stop stack and remove volumes.
 Example: `make up && make airflow-init` then open `http://localhost:8080`.
 
+## Локальное Python-окружение
+- Окружением управляет `uv`: `uv python install 3.11` и `uv python pin 3.11` скачивают и фиксируют версию Python для проекта.
+- `uv sync` (или `make dev-setup` / `make dev-sync`) создаёт `.venv` и ставит dev-зависимости из `pyproject.toml` / `uv.lock`.
+- Команды разработчика: `make test`, `make lint`, `make fmt` (под капотом выполняются через `uv run`).
+- Не используем `pip install --user`; если пакеты попали в user-site, удаляем через `pip uninstall <package>` и проверяем `pip list --user`.
+- В IDE выбираем интерпретатор из `.venv` (`.venv\Scripts\python.exe` на Windows, `.venv/bin/python` на Linux/macOS).
+
 ## Coding Style & Naming Conventions
 - Python: PEP 8, 4-space indents, `snake_case` for functions/vars, DAG IDs lower_snake_case.
 - Imports: stdlib → third-party → local; prefer one module per line.

airflow-greenplum/Makefile

@@ -1,4 +1,8 @@
 SHELL := /bin/bash
+UV := uv
+PYTHON_VERSION := 3.11
+
+.PHONY: up down airflow-init logs gp-psql ddl-gp dev-setup dev-sync dev-lock test lint fmt clean-venv
 
 up:
 	docker compose -f docker-compose.yml up -d
@@ -17,3 +21,28 @@ gp-psql:
 
 ddl-gp:
 	docker compose -f docker-compose.yml exec greenplum bash -c "su - gpadmin -c '/usr/local/greenplum-db/bin/psql -d gpadmin -f /sql/ddl_gp.sql'"
+
+dev-setup:
+	$(UV) python install $(PYTHON_VERSION)
+	$(UV) python pin $(PYTHON_VERSION)
+	$(UV) sync
+
+dev-sync:
+	$(UV) sync
+
+dev-lock:
+	$(UV) lock --upgrade
+
+test:
+	$(UV) run pytest -q
+
+lint:
+	$(UV) run black --check airflow tests
+	$(UV) run isort --check-only airflow tests
+
+fmt:
+	$(UV) run black airflow tests
+	$(UV) run isort airflow tests
+
+clean-venv:
+	python -c "import shutil; shutil.rmtree('.venv', ignore_errors=True)"

airflow-greenplum/README.md

@@ -1,4 +1,4 @@
-# DE Starter Kit — Airflow + Greenplum + CSV
+# DE Starter Kit — Airflow + Greenplum + CSV
 
 Добро пожаловать в учебный стенд для изучения основ Data Engineering! Этот проект поможет вам освоить ключевые инструменты современных data pipeline: **Airflow** для оркестрации, **pandas/CSV** для подготовки данных и **Greenplum** как аналитическую базу данных.
 
@@ -10,7 +10,27 @@
 - Как загружать данные в Greenplum пакетами и избегать дублей
 - Как проверять качество данных в автоматизированных pipeline
 - Основы проектирования ETL/ELT процессов
+## Локальное окружение разработчика
 
+Локальным окружением управляет [uv](https://docs.astral.sh/uv/) — он скачивает нужный Python и создаёт `.venv` на основе `pyproject.toml` / `uv.lock`.
+
+```bash
+uv python install 3.11      # один раз загружаем CPython 3.11
+uv python pin 3.11          # фиксируем версию для проекта (.python-version)
+uv sync                     # создаём .venv и ставим dev-зависимости
+```
+
+Можно короче: `make dev-setup`. После изменения зависимостей запустите `uv sync` (или `make dev-sync`).
+
+Проверки и форматирование выполняем через uv:
+
+```bash
+make test            # uv run pytest -q
+make lint            # black/isort в режиме проверки
+make fmt             # автоформатирование black + isort
+```
+
+> Не устанавливайте пакеты напрямую через `pip install --user ...`. Если что-то уже попало в user-site, удалите `pip uninstall <package>` и проверьте `pip list --user`.
 ---
 
 ## 🚀 Быстрый старт (для новичков)
@@ -212,3 +232,4 @@ make logs               # Следить за логами Airflow
 4. **Изучите Airflow deeper** — добавьте зависимости между задачами, настройте расписания
 
 Удачи в изучении Data Engineering! 🚀
+

airflow-greenplum/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "airflow-greenplum"
+version = "0.1.0"
+requires-python = ">=3.11,<3.12"
+dependencies = []
+
+[dependency-groups]
+dev = [
+    "apache-airflow==2.9.2",
+    "apache-airflow-providers-postgres==5.11.1",
+    "black==24.4.2",
+    "isort>=7.0.0",
+    "pandas==2.1.4",
+    "psycopg2-binary==2.9.9",
+    "pytest==7.4.4",
+]

airflow-greenplum/tests/conftest.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import importlib
+import sys
+from pathlib import Path
+from types import ModuleType
+from typing import Type
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.append(str(PROJECT_ROOT))
+
+if "airflow" not in sys.modules:
+    airflow_module = ModuleType("airflow")
+    airflow_module.__path__ = [str(PROJECT_ROOT / "airflow")]
+    sys.modules["airflow"] = airflow_module
+
+    providers_module = ModuleType("airflow.providers")
+    providers_module.__path__ = []
+    sys.modules["airflow.providers"] = providers_module
+    airflow_module.providers = providers_module
+
+    postgres_module = ModuleType("airflow.providers.postgres")
+    postgres_module.__path__ = []
+    sys.modules["airflow.providers.postgres"] = postgres_module
+    providers_module.postgres = postgres_module
+
+    hooks_module = ModuleType("airflow.providers.postgres.hooks")
+    hooks_module.__path__ = []
+    sys.modules["airflow.providers.postgres.hooks"] = hooks_module
+    postgres_module.hooks = hooks_module
+
+if "psycopg2" not in sys.modules:
+    psycopg2_stub = ModuleType("psycopg2")
+    psycopg2_stub.connect = lambda **_: None  # type: ignore[assignment]
+    sys.modules["psycopg2"] = psycopg2_stub
+
+
+def _ensure_stub_module(full_name: str) -> ModuleType:
+    """
+    Ensure that module placeholders exist for a dotted path and return leaf module.
+    """
+    parts = full_name.split(".")
+    module: ModuleType | None = None
+    path = ""
+    for part in parts:
+        path = f"{path}.{part}" if path else part
+        if path not in sys.modules:
+            new_module = ModuleType(path)
+            if module is not None:
+                setattr(module, part, new_module)
+            sys.modules[path] = new_module
+            module = new_module
+        else:
+            module = sys.modules[path]
+    assert isinstance(module, ModuleType)
+    return module
+
+
+def patch_postgres_hook(monkeypatch, hook_cls: Type) -> None:
+    """
+    Patch PostgresHook so that helpers.greenplum can be exercised without real Airflow.
+    """
+    try:
+        module = importlib.import_module("airflow.providers.postgres.hooks.postgres")
+    except ModuleNotFoundError:
+        module = _ensure_stub_module("airflow.providers.postgres.hooks.postgres")
+    monkeypatch.setattr(module, "PostgresHook", hook_cls, raising=False)

airflow-greenplum/tests/test_greenplum_helpers.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, List, Sequence
+
+import pytest
+
+import airflow.dags.helpers.greenplum as greenplum
+from tests.conftest import patch_postgres_hook
+
+
+@dataclass
+class FakeCursor:
+    fetchone_value: Any = None
+    fetchall_value: Sequence[Any] | None = None
+    rowcount: int | None = None
+
+    def __post_init__(self) -> None:
+        self.queries: List[Any] = []
+
+    def execute(self, query: str, params: Any | None = None) -> None:
+        self.queries.append((query, params))
+
+    def fetchone(self) -> Any:
+        return self.fetchone_value
+
+    def fetchall(self) -> Sequence[Any] | None:
+        return self.fetchall_value
+
+    def __enter__(self) -> FakeCursor:
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        return None
+
+
+class FakeConn:
+    def __init__(self, cursors: Sequence[FakeCursor]) -> None:
+        self._cursors = list(cursors)
+        self._index = 0
+        self.commits = 0
+
+    def cursor(self) -> FakeCursor:
+        cursor = self._cursors[self._index]
+        self._index += 1
+        return cursor
+
+    def commit(self) -> None:
+        self.commits += 1
+
+
+def test_get_gp_conn_uses_airflow_hook(monkeypatch) -> None:
+    class FakeHook:
+        def __init__(self, postgres_conn_id: str) -> None:
+            self.postgres_conn_id = postgres_conn_id
+
+        def get_conn(self) -> str:
+            return "hook_connection"
+
+    patch_postgres_hook(monkeypatch, FakeHook)
+    monkeypatch.setattr(greenplum, "GP_CONN_ID", "demo_conn", raising=False)
+    monkeypatch.setattr(greenplum, "GP_USE_AIRFLOW_CONN", True, raising=False)
+
+    conn = greenplum.get_gp_conn()
+
+    assert conn == "hook_connection"
+
+
+def test_get_gp_conn_fallback_to_psycopg(monkeypatch) -> None:
+    class BrokenHook:
+        def __init__(self, postgres_conn_id: str) -> None:
+            self.postgres_conn_id = postgres_conn_id
+
+        def get_conn(self):
+            raise RuntimeError("boom")
+
+    patch_postgres_hook(monkeypatch, BrokenHook)
+    monkeypatch.setattr(greenplum, "GP_USE_AIRFLOW_CONN", True, raising=False)
+    monkeypatch.setattr(greenplum, "GP_CONN_ID", "demo_conn", raising=False)
+    monkeypatch.setenv("GP_DB", "demo_db")
+    monkeypatch.setenv("GP_USER", "demo_user")
+    monkeypatch.setenv("GP_PASSWORD", "secret")
+    monkeypatch.setenv("GP_HOST", "greenplum-host")
+    monkeypatch.setenv("GP_PORT", "5434")
+
+    captured_kwargs = {}
+
+    def fake_connect(**kwargs):
+        captured_kwargs.update(kwargs)
+        return "psycopg_connection"
+
+    monkeypatch.setattr(greenplum.psycopg2, "connect", fake_connect)
+
+    conn = greenplum.get_gp_conn()
+
+    assert conn == "psycopg_connection"
+    assert captured_kwargs == {
+        "dbname": "demo_db",
+        "user": "demo_user",
+        "password": "secret",
+        "host": "greenplum-host",
+        "port": 5434,
+    }
+
+
+def test_get_gp_conn_without_airflow(monkeypatch) -> None:
+    monkeypatch.setattr(greenplum, "GP_USE_AIRFLOW_CONN", False, raising=False)
+    monkeypatch.setenv("GP_DB", "demo_db")
+    monkeypatch.setenv("GP_USER", "demo_user")
+    monkeypatch.setenv("GP_PASSWORD", "secret")
+    monkeypatch.setenv("GP_HOST", "greenplum-host")
+    monkeypatch.setenv("GP_PORT", "5435")
+
+    captured_kwargs = {}
+
+    def fake_connect(**kwargs):
+        captured_kwargs.update(kwargs)
+        return "direct_psycopg"
+
+    monkeypatch.setattr(greenplum.psycopg2, "connect", fake_connect)
+
+    conn = greenplum.get_gp_conn()
+
+    assert conn == "direct_psycopg"
+    assert captured_kwargs["port"] == 5435
+
+
+def test_assert_orders_table_exists_ok() -> None:
+    conn = FakeConn([FakeCursor(fetchone_value=(1,))])
+
+    greenplum.assert_orders_table_exists(conn)
+
+
+def test_assert_orders_table_exists_missing() -> None:
+    conn = FakeConn([FakeCursor(fetchone_value=None)])
+
+    with pytest.raises(ValueError):
+        greenplum.assert_orders_table_exists(conn)
+
+
+def test_assert_orders_schema_ok() -> None:
+    expected = list(greenplum.EXPECTED_ORDERS_SCHEMA)
+    conn = FakeConn([FakeCursor(fetchall_value=expected)])
+
+    greenplum.assert_orders_schema(conn)
+
+
+def test_assert_orders_schema_mismatch() -> None:
+    conn = FakeConn([FakeCursor(fetchall_value=[("order_id", "bigint")])])
+
+    with pytest.raises(ValueError):
+        greenplum.assert_orders_schema(conn)
+
+
+def test_assert_orders_have_rows_ok() -> None:
+    conn = FakeConn([FakeCursor(fetchone_value=(5,))])
+
+    greenplum.assert_orders_have_rows(conn)
+
+
+def test_assert_orders_have_rows_empty() -> None:
+    conn = FakeConn([FakeCursor(fetchone_value=(0,))])
+
+    with pytest.raises(ValueError):
+        greenplum.assert_orders_have_rows(conn)
+
+
+def test_assert_orders_no_duplicates_ok() -> None:
+    conn = FakeConn([FakeCursor(fetchone_value=(0,))])
+
+    greenplum.assert_orders_no_duplicates(conn)
+
+
+def test_assert_orders_no_duplicates_detected() -> None:
+    conn = FakeConn([FakeCursor(fetchone_value=(3,))])
+
+    with pytest.raises(ValueError):
+        greenplum.assert_orders_no_duplicates(conn)