From 29df0422714db6325661416bc993753741f7ef98 Mon Sep 17 00:00:00 2001 From: Dmitry Dementev Date: Mon, 3 Nov 2025 21:51:51 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A4=D0=B8=D0=BA=D1=81=20=D1=81=D0=B1=D0=BE?= =?UTF-8?q?=D1=80=D0=BA=D0=B8=20airflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 43 +++++++++++++++++++++++++++++++ airflow-docker/Dockerfile | 11 +++----- airflow-docker/data/.gitkeep | 0 airflow-docker/docker-compose.yml | 16 ++++++++---- airflow-docker/requirements.txt | 4 +-- 5 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 AGENTS.md create mode 100644 airflow-docker/data/.gitkeep diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..51c5270 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,43 @@ +# Repository Guidelines + +This repository contains an educational manual for Apache Airflow with runnable examples. The root holds the written guides; `airflow-docker/` provides a self-contained Docker setup to run and explore DAGs. + +## Project Structure & Module Organization +- Root `01-09 *.md`: step-by-step articles (RU). +- `_attachments/`: images and GIFs used by the docs. +- `airflow-docker/`: Dockerized Airflow environment. + - `docker-compose.yml`, `Dockerfile`, `requirements.txt`. + - `dags/`: Python DAG examples (e.g., `hello_world_dag.py`). + - `data/`: sample input data for exercises. +- Note: `airflow-docker/AGENTS.md` adds extra, folder-specific guidance and takes precedence there. + +## Build, Test, and Development Commands +Prerequisite: Docker + Docker Compose. +- Build images: `cd airflow-docker && docker-compose build` +- Initialize Airflow DB and admin: `docker-compose run --rm airflow-init` +- Start services: `docker-compose up -d airflow-webserver airflow-scheduler` +- Web UI: http://localhost:8080 (admin/admin) +- List DAGs: `docker-compose exec airflow-webserver airflow dags list` +- Follow scheduler logs: `docker-compose logs -f airflow-scheduler` +- Stop: `docker-compose down` (add `-v` to drop volumes; this deletes data). + +## Coding Style & Naming Conventions +- Language: Python 3; follow PEP 8; 4-space indentation. +- DAG files: name as `*_dag.py` (e.g., `data_processing_dag.py`). +- Names: `snake_case` for functions/variables/tasks, `UPPER_CASE` for constants. +- Structure: keep imports grouped (stdlib, third-party, local). Prefer docstrings and clear task IDs. + +## Testing Guidelines +- Quick checks via Airflow CLI inside containers: + - Task dry-run: `docker-compose exec airflow-webserver airflow tasks test 2024-01-01` + - Validate DAGs load: `docker-compose exec airflow-webserver airflow dags list` +- For docs, ensure referenced files exist under `_attachments/` and paths render correctly. + +## Commit & Pull Request Guidelines +- Commits: short, imperative summary (≤72 chars), optional body explaining why/how. Examples: “Добавить DAG для ветвления”, “Документация: обновить раздел Gantt”. +- Reference issues with `#` when relevant. +- PRs: clear description, steps to validate locally, affected DAGs/docs, and screenshots for doc/UI changes. Keep PRs focused on one logical change. + +## Security & Configuration Tips +- Do not commit secrets. Use environment variables and local `.env` files if needed. +- Use `airflow-docker/data/` for sample data; avoid real PII in the repo. diff --git a/airflow-docker/Dockerfile b/airflow-docker/Dockerfile index 336fa21..08bed08 100644 --- a/airflow-docker/Dockerfile +++ b/airflow-docker/Dockerfile @@ -4,11 +4,11 @@ FROM apache/airflow:2.9.2 # Set environment variables ENV AIRFLOW_HOME=/opt/airflow -# Copy the optimized requirements file -COPY requirements-optimized.txt . +# Copy the project requirements file +COPY --chown=airflow:0 requirements.txt ${AIRFLOW_HOME}/requirements.txt # Install the required Python packages -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt # Create necessary directories RUN mkdir -p /opt/airflow/dags /opt/airflow/logs /opt/airflow/config /opt/airflow/data @@ -16,11 +16,8 @@ RUN mkdir -p /opt/airflow/dags /opt/airflow/logs /opt/airflow/config /opt/airflo # Set working directory WORKDIR ${AIRFLOW_HOME} -# Change ownership to airflow user -RUN chown -R airflow:airflow ${AIRFLOW_HOME} - # Expose the webserver port EXPOSE 8080 # The image will use Airflow's default entrypoint -# Commands will be passed at runtime \ No newline at end of file +# Commands will be passed at runtime diff --git a/airflow-docker/data/.gitkeep b/airflow-docker/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/airflow-docker/docker-compose.yml b/airflow-docker/docker-compose.yml index 9da968e..2ef49fc 100644 --- a/airflow-docker/docker-compose.yml +++ b/airflow-docker/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: # PostgreSQL for Airflow metadata postgres-metadata: @@ -44,6 +42,7 @@ services: AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string command: > bash -c " airflow webserver @@ -67,6 +66,7 @@ services: AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string command: > bash -c " airflow scheduler @@ -83,17 +83,23 @@ services: # Airflow init to create admin user airflow-init: build: . + user: "0:0" # image: airflow-optimized:2.9.2 environment: AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-metadata:5432/airflow + AIRFLOW__WEBSERVER__SECRET_KEY: replace-me-with-random-string volumes: - ./dags:/opt/airflow/dags - ./data:/opt/airflow/data command: > - bash -c " - airflow db migrate && - airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org + bash -ceuo pipefail " + mkdir -p /opt/airflow/data && + chmod -R 777 /opt/airflow/data || true && + chown -R airflow:0 /opt/airflow/data || true && + umask 000 && + su -s /bin/bash airflow -c 'airflow db migrate' && + su -s /bin/bash airflow -c 'airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.org' " depends_on: postgres-metadata: diff --git a/airflow-docker/requirements.txt b/airflow-docker/requirements.txt index 05cd212..4c258b4 100644 --- a/airflow-docker/requirements.txt +++ b/airflow-docker/requirements.txt @@ -2,10 +2,10 @@ # Only includes packages actually used in DAGs # Core database connector -psycopg2-binary==2.9.7 +psycopg2-binary==2.9.9 # Data processing (used in data_processing_dag.py and file_operations_dag.py) pandas==2.1.4 # Airflow PostgreSQL provider (used in sql_basic_dag.py and data_processing_dag.py) -apache-airflow-providers-postgres==8.5.1 \ No newline at end of file +apache-airflow-providers-postgres==5.11.1