From aa31d4b9bba600cda1f757c70c92a3f385fdf330 Mon Sep 17 00:00:00 2001 From: Dmitry Dementev Date: Mon, 3 Nov 2025 20:55:54 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9E=D1=82=D0=BB=D0=B0=D0=B4=D0=BA=D0=B0=20?= =?UTF-8?q?=D0=B4=D0=BE=D1=81=D1=82=D1=83=D0=BF=D0=B0=20=D0=BA=20/opt/airf?= =?UTF-8?q?low/data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 8 +------- README.md | 5 +---- docker-compose.yml | 11 ++++++++--- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 3764822..bc853ed 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL := /bin/bash UV := uv PYTHON_VERSION := 3.11 -.PHONY: up down start stop airflow-init logs gp-psql ddl-gp dev-setup dev-sync dev-lock test lint fmt clean-venv +.PHONY: up down airflow-init logs gp-psql ddl-gp dev-setup dev-sync dev-lock test lint fmt clean-venv up: docker compose -f docker-compose.yml up -d @@ -10,12 +10,6 @@ up: down: docker compose -f docker-compose.yml down -v -start: - docker compose -f docker-compose.yml start - -stop: - docker compose -f docker-compose.yml stop - airflow-init: docker compose -f docker-compose.yml run --rm airflow-init diff --git a/README.md b/README.md index 48ab7d6..22d8d5f 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,6 @@ cp .env.example .env # Запустите стек (это может занять 2-3 минуты при первом запуске) docker compose up -d - -# Инициализируйте Airflow -docker compose run --rm airflow-init ``` ### Шаг 3: Первый запуск pipeline @@ -81,7 +78,7 @@ CSV-файлы после выполнения DAG остаются в дире ```bash make down # Остановить и стереть данные в контейнерах -make up && make airflow-init +make up ``` Это помогает, когда Greenplum не стартует из‑за «грязной» остановки и внутренних файлов. diff --git a/docker-compose.yml b/docker-compose.yml index 8d4a58e..56f1afd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: volumes: - ./sql:/sql:ro - greenplum_data:/data + - airflow_data:/opt/airflow/data # Простая проверка доступности: psql откликается healthcheck: test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -p 5432 -U ${GP_USER:-gpadmin} -d ${GP_DB:-gpadmin} || echo 1"] @@ -56,7 +57,7 @@ services: volumes: - ./airflow/dags:/opt/airflow/dags - ./airflow/requirements.txt:/opt/airflow/requirements.txt - - ./data:/opt/airflow/data + - airflow_data:/opt/airflow/data depends_on: pgmeta: condition: service_healthy @@ -77,7 +78,7 @@ services: volumes: - ./airflow/dags:/opt/airflow/dags - ./airflow/requirements.txt:/opt/airflow/requirements.txt - - ./data:/opt/airflow/data + - airflow_data:/opt/airflow/data depends_on: pgmeta: condition: service_healthy @@ -92,13 +93,16 @@ services: AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} + user: "0:0" volumes: - ./airflow/dags:/opt/airflow/dags - ./airflow/requirements.txt:/opt/airflow/requirements.txt - - ./data:/opt/airflow/data + - airflow_data:/opt/airflow/data command: > bash -lc " set -e; + # mkdir -p /opt/airflow/data && chmod -R 777 /opt/airflow/data; + # su - airflow; pip install --no-cache-dir -r /opt/airflow/requirements.txt; # Дожидаемся готовности БД ретрая миграции for i in {1..30}; do @@ -114,3 +118,4 @@ services: volumes: pgmeta: greenplum_data: + airflow_data: