diff --git a/.env.example b/.env.example index 64e15b5..c093ed7 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,6 @@ +# Общие настройки +TZ=Europe/Moscow + # Airflow Configuration AIRFLOW_USER=admin AIRFLOW_PASSWORD=admin diff --git a/TESTING.md b/TESTING.md index c07262b..23324ab 100644 --- a/TESTING.md +++ b/TESTING.md @@ -42,6 +42,8 @@ - `SELECT COUNT(*) FROM public.orders;` — оценка объёма. - `SELECT * FROM public.orders LIMIT 5;` — визуальная проверка. - `SELECT order_id FROM public.orders GROUP BY 1 HAVING COUNT(*) > 1;` — поиск дублей. + - (после настройки PXF) `SELECT COUNT(*) FROM public.ext_bookings_bookings;` — проверка чтения из демо-БД bookings через PXF. + - (после настройки PXF) `SELECT * FROM public.ext_bookings_bookings LIMIT 5;` — визуальное сравнение с таблицей `bookings.bookings` в исходной БД. - Завершить `\q`. ## 6. Негативные сценарии и fallback diff --git a/docker-compose.yml b/docker-compose.yml index 3173f54..45ebafe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ services: # container_name: gp_pgmeta env_file: .env environment: + TZ: ${TZ:-Europe/Moscow} POSTGRES_USER: ${PG_USER} POSTGRES_PASSWORD: ${PG_PASSWORD} POSTGRES_DB: ${PG_DB} @@ -23,6 +24,7 @@ services: image: postgres:16 env_file: .env environment: + TZ: ${TZ:-Europe/Moscow} POSTGRES_USER: ${BOOKINGS_DB_USER} POSTGRES_PASSWORD: ${BOOKINGS_DB_PASSWORD} POSTGRES_DB: ${BOOKINGS_DB_NAME} @@ -43,15 +45,19 @@ services: # container_name: gp_single # hostname: gpdbsne environment: + TZ: ${TZ:-Europe/Moscow} GREENPLUM_USER: ${GP_USER:-gpadmin} GREENPLUM_PASSWORD: ${GP_PASSWORD:-gpadmin} GREENPLUM_DATABASE_NAME: ${GP_DB:-gpadmin} + GREENPLUM_PXF_ENABLE: "true" # GP_PORT: ${GP_PORT:-5432} # Порты: внешний 5432 ports: - "${GP_PORT}:5432" volumes: - ./sql:/sql:ro + - ./pxf/postgresql-42.7.3.jar:/data/pxf/lib/postgresql-jdbc.jar:ro + - ./pxf/servers/bookings-db/jdbc-site.xml:/data/pxf/servers/bookings-db/jdbc-site.xml:ro - greenplum_data:/data - airflow_data:/opt/airflow/data # Простая проверка доступности: psql откликается @@ -66,6 +72,7 @@ services: container_name: gp_airflow_web env_file: .env environment: + TZ: ${TZ:-Europe/Moscow} AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} @@ -89,6 +96,7 @@ services: container_name: gp_airflow_sch env_file: .env environment: + TZ: ${TZ:-Europe/Moscow} AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} @@ -110,6 +118,7 @@ services: # container_name: gp_airflow_init env_file: .env environment: + TZ: ${TZ:-Europe/Moscow} AIRFLOW__CORE__LOAD_EXAMPLES: "False" AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER}:${PG_PASSWORD}@pgmeta:5432/${PG_DB} AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY} diff --git a/pxf/postgresql-42.7.3.jar b/pxf/postgresql-42.7.3.jar new file mode 100644 index 0000000..fa42b1d Binary files /dev/null and b/pxf/postgresql-42.7.3.jar differ diff --git a/pxf/servers/bookings-db/jdbc-site.xml b/pxf/servers/bookings-db/jdbc-site.xml new file mode 100644 index 0000000..dca82e9 --- /dev/null +++ b/pxf/servers/bookings-db/jdbc-site.xml @@ -0,0 +1,31 @@ + + + + + + jdbc.driver + org.postgresql.Driver + + + + jdbc.url + jdbc:postgresql://bookings-db:5432/demo + + + + jdbc.user + bookings + + + + jdbc.password + bookings + + + + diff --git a/sql/ddl_gp.sql b/sql/ddl_gp.sql index 1047666..b0ae2bf 100644 --- a/sql/ddl_gp.sql +++ b/sql/ddl_gp.sql @@ -10,3 +10,13 @@ CREATE TABLE IF NOT EXISTS public.orders ( ) WITH (appendonly=true, orientation=row, compresstype=zlib, compresslevel=1) DISTRIBUTED BY (order_id); + +-- Внешняя таблица для чтения данных из демо-БД bookings через PXF (JDBC). +-- Источник: таблица bookings.bookings в базе demo (Postgres, сервис bookings-db). +CREATE EXTERNAL TABLE IF NOT EXISTS public.ext_bookings_bookings ( + book_ref CHAR(6), + book_date TIMESTAMP WITH TIME ZONE, + total_amount NUMERIC(10,2) +) +LOCATION ('pxf://bookings.bookings?PROFILE=JDBC&SERVER=bookings-db') +FORMAT 'CUSTOM' (formatter='pxfwritable_import');