diff --git a/airflow-docker/AGENTS.md b/airflow-docker/AGENTS.md new file mode 100644 index 0000000..6803d8e --- /dev/null +++ b/airflow-docker/AGENTS.md @@ -0,0 +1,187 @@ +# AGENTS.md - Guide for AI Agents + +## 🎯 Purpose +This file provides guidance for AI agents working on the educational Airflow setup. It contains information about code style, debugging approaches, and project structure. + +## 📁 Project Structure Overview + +``` +airflow-docker/ +├── docker-compose.yml # Docker configuration +├── .env # Hardcoded environment variables +├── dags/ # DAG files for learning +│ ├── hello_world_dag.py # Basic Python operators +│ ├── sql_basic_dag.py # SQL operations +│ ├── file_operations_dag.py # File processing +│ ├── data_processing_dag.py # ETL pipeline +│ ├── branching_dag.py # Conditional logic +│ └── error_handling_dag.py # Error handling +├── data/ # Data for exercises +│ ├── input/ # Input data +│ └── output/ # Processing results +├── logs/ # Airflow logs +└── README.md # Main documentation +``` + +## 🎓 Learning Progression for Students + +### Basic Concepts +- **DAG Structure**: Understanding basic DAG components +- **Python Operators**: PythonOperator, BashOperator basics +- **Task Dependencies**: Setting up task execution order + +### Data Integration +- **PostgreSQL Connections**: Database connectivity +- **SQL Operations**: CRUD operations in tasks +- **File Processing**: CSV operations and data transformation + +### Advanced Features +- **Conditional Logic**: BranchPythonOperator usage +- **Error Handling**: Task retries and failure management + +## 🔧 Code Style Guidelines + +### Python Code Style +- Use 4-space indentation +- Follow PEP 8 conventions +- Include comprehensive docstrings in Russian +- Use descriptive variable names in English + +### DAG File Structure +```python +""" +Описание DAG на русском языке +Уровень: Начальный/Средний/Продвинутый +""" +from datetime import datetime, timedelta +from airflow import DAG +from airflow.operators.python import PythonOperator + +# Default arguments for DAG +default_args = { + 'owner': 'student', + 'depends_on_past': False, + 'start_date': datetime(2023, 1, 1), + 'email_on_failure': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5) +} + +# DAG definition +dag = DAG( + 'example_dag', + default_args=default_args, + description='Описание функциональности DAG', + schedule_interval=timedelta(days=1), + catchup=False, + tags=['educational', 'beginner'] +) +``` + +## 🐛 Debugging Approaches + +### Common Issues and Solutions + +#### 1. DAG Not Appearing in UI +- Check DAG file location (`dags/` directory) +- Verify Python syntax and imports +- Check scheduler logs: `docker-compose logs airflow-scheduler` +- Ensure DAG has valid start_date + +#### 2. Database Connection Errors +- Verify PostgreSQL services are running +- Check connection strings in environment variables +- Confirm database health checks + +#### 3. Task Failures +- Check task logs in Airflow UI +- Verify required Python packages are installed +- Check database permissions and credentials + +#### 4. Import Errors +- Ensure all required imports are available +- Check `requirements.txt` for missing dependencies + +### Log Analysis +```bash +# Airflow scheduler logs +docker-compose logs airflow-scheduler + +# Airflow webserver logs +docker-compose logs airflow-webserver + +# PostgreSQL logs +docker-compose logs postgres-training +docker-compose logs postgres-metadata +``` + +## 📚 Key Files to Examine + +### Configuration Files +- [`docker-compose.yml`](docker-compose.yml) - Main Docker configuration +- [`.env`](.env) - Environment variables +- [`requirements.txt`](requirements.txt) - Python dependencies + +### Sample Data Files +- [`customers.csv`](data/input/customers.csv) - Customer data for exercises +- [`orders.csv`](data/input/orders.csv) - Order data for practical work + +## 🛠️ Development Workflow + +### Adding New DAGs +1. Create Python file in `dags/` directory +2. Ensure proper DAG structure and imports +3. File will be automatically discovered by scheduler + +### Testing Changes +1. Restart services after major changes +2. Monitor scheduler logs for DAG processing +3. Use Airflow UI for monitoring and debugging + +## 💡 Best Practices for AI Agents + +### When Making Changes +- Always read the file first using `read_file` +- Use `apply_diff` for surgical edits +- Test DAG execution in Airflow UI + +## 🎓 Educational Focus Areas + +### For Beginners +- Focus on clear, commented code +- Include expected output descriptions +- Provide common pitfalls and solutions + +### Code Quality Checks +- Validate DAG structure before deployment +- Test with sample data first +- Provide clear error messages and handling + +### Progressive Complexity +- Start with simple print statements +- Progress to database operations +- Advance to error handling and conditional logic + +## 🔍 Troubleshooting Checklist + +### Before Reporting Issues +- [ ] Services are running: `docker-compose ps` +- [ ] DAG files are in correct location +- [ ] Environment variables are properly set +- [ ] Database connections are established +- [ ] Task dependencies are correctly set +- [ ] Required Python packages are installed +- [ ] DAG syntax is correct +- [ ] No import errors in DAG files + +### Performance Monitoring +- Check task execution times in Airflow UI +- Monitor resource usage in Docker +- Review logs for warnings or errors + +## 📝 Documentation Standards + +### For New DAGs +- Include comprehensive docstring in Russian +- Describe learning objectives clearly +- Provide step-by-step task explanations \ No newline at end of file diff --git a/airflow-docker/CHANGELOG.md b/airflow-docker/CHANGELOG.md new file mode 100644 index 0000000..4ed9e1c --- /dev/null +++ b/airflow-docker/CHANGELOG.md @@ -0,0 +1,70 @@ +# CHANGELOG - Educational Airflow Setup + +## 📋 Summary of Changes Made + +### 1. Fixed Docker Compose Structure +- Removed duplicate `services:` sections +- Added missing service definitions (postgres-metadata, postgres-training, airflow-webserver, airflow-scheduler, airflow-init) +- Configured proper service dependencies and health checks + +### 2. Created Hardcoded Environment Variables +- `.env` file with all necessary credentials: + - Airflow: admin/admin + - PostgreSQL metadata: airflow/airflow + - PostgreSQL training: student/student + +### 3. Implemented Educational DAG Examples + +**Level 1: Basic Concepts** +- `hello_world_dag.py` - Simple PythonOperator tasks with dependencies + +**Level 2: Intermediate Integration** +- `sql_basic_dag.py` - PostgreSQL operations +- `file_operations_dag.py` - File processing and CSV operations + +**Level 3: Advanced Features** +- `data_processing_dag.py` - ETL pipeline with multiple steps + +**Level 4: Conditional Logic** +- `branching_dag.py` - BranchPythonOperator and conditional execution + +**Level 5: Error Handling** +- `error_handling_dag.py` - Task retries and error management + +### 4. Sample Data Files +- `customers.csv` - Customer data for exercises +- `orders.csv` - Order data for practical work + +### 5. Database Configuration +- PostgreSQL for Airflow metadata (port 5433) +- PostgreSQL for training exercises (port 5432) + +### 6. Configuration Files +- `requirements.txt` - Essential Python packages +- Updated `README.md` with comprehensive setup instructions + +## 🎯 Key Features for Beginners + +### Simplified Setup +- All environment variables hardcoded in `.env` +- Clear port mappings and access credentials + +### Progressive Learning +- From basic "Hello World" to advanced ETL pipelines +- Hands-on exercises with real data + +### Technical Specifications +- Airflow version: 2.9.2 +- PostgreSQL version: 16 +- LocalExecutor for simplicity + +## 🚀 Quick Start Commands + +```bash +# Start all services +docker-compose up -d + +# Access interfaces +- Airflow UI: http://localhost:8080 +- Training PostgreSQL: localhost:5432 +- Metadata PostgreSQL: localhost:5433 \ No newline at end of file diff --git a/airflow-docker/README.md b/airflow-docker/README.md index 75a75f1..643a3d7 100644 --- a/airflow-docker/README.md +++ b/airflow-docker/README.md @@ -14,24 +14,17 @@ ## 🚀 Быстрый старт -### 1. Клонирование и настройка +### 1. Запуск стенда ```bash # Перейдите в директорию проекта cd airflow-docker -# Создайте необходимые директории -mkdir -p dags data/input data/output logs -``` - -### 2. Запуск стенда - -```bash # Запустите все сервисы docker-compose up -d ``` -### 3. Доступ к интерфейсам +### 2. Доступ к интерфейсам - **Airflow UI**: http://localhost:8080 - Логин: `admin` @@ -42,7 +35,7 @@ docker-compose up -d - Пользователь: `student` - Пароль: `student` -- **PostgreSQL для метаданных Airflow**: `localhost:5434` +- **PostgreSQL для метаданных Airflow**: `localhost:5433` - База данных: `airflow` - Пользователь: `airflow` - Пароль: `airflow` @@ -82,7 +75,9 @@ airflow-docker/ │ ├── hello_world_dag.py # Базовый пример │ ├── sql_basic_dag.py # Работа с SQL │ ├── file_operations_dag.py # Обработка файлов -│ └── data_processing_dag.py # ETL пайплайн +│ ├── data_processing_dag.py # ETL пайплайн +│ ├── branching_dag.py # Условная логика +│ └── error_handling_dag.py # Обработка ошибок ├── data/ # Данные для упражнений │ ├── input/ # Входные данные │ └── output/ # Результаты обработки @@ -132,19 +127,32 @@ airflow-docker/ Все переменные захардкожены для простоты: ```env -# Airflow +# Airflow Configuration AIRFLOW_USER=admin AIRFLOW_PASSWORD=admin -# PostgreSQL для метаданных Airflow -POSTGRES_USER=airflow -POSTGRES_PASSWORD=airflow -POSTGRES_DB=airflow +# PostgreSQL (Airflow metadata) +PG_USER=airflow +PG_PASSWORD=airflow +PG_DB=airflow -# PostgreSQL для учебных упражнений -POSTGRES_USER=student -POSTGRES_PASSWORD=student -POSTGRES_DB=training +# PostgreSQL для training exercises +TRAINING_PG_USER=student +TRAINING_PG_PASSWORD=student +TRAINING_PG_DB=training +TRAINING_PG_HOST=postgres-training +TRAINING_PG_PORT=5432 + +# Connection ID for PostgreSQL training database +POSTGRES_CONN_ID=postgres_training + +# CSV Pipeline +CSV_DIR=/opt/airflow/data +CSV_ROWS=1000 + +# Airflow Configuration +_AIRFLOW_DB_UPGRADE=true +_AIRFLOW_WWW_USER_CREATE=true ``` ### Порты