From 62dc618252932933eb508cb8b6b678ec175c2831 Mon Sep 17 00:00:00 2001 From: Dmitry Dementiev Date: Thu, 13 Aug 2026 10:19:38 +0300 Subject: [PATCH] =?UTF-8?q?feat(clickhouse):=20=D0=BF=D1=80=D0=B8=D0=BB?= =?UTF-8?q?=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BF=D0=B5=D1=80=D0=B5?= =?UTF-8?q?=D0=BA=D0=BB=D1=8E=D1=87=D0=B5=D0=BD=D1=8B=20=D0=BD=D0=B0=20?= =?UTF-8?q?=D1=80=D0=BE=D0=BB=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Зачем: - DDL, Airflow и Superset не должны работать с правами default. - Что: - DDL и Airflow переведены на etl, включая явный доступ remote(). - Superset переведён на bi с паролем из окружения. - etl получил право KAFKA, которое ClickHouse 26.3 требует для движка. - Проверка: - make config-test, make lint, make up, make smoke, make check-services. --- compose.yaml | 5 +++ dags/test_clickhouse.py | 40 +++++++++++++++---- infra/airflow/init.sh | 3 +- infra/clickhouse/users.d/access.xml | 2 + .../superset/import/databases/clickhouse.yaml | 2 +- infra/superset/superset_config.py | 9 +++++ scripts/stand-services.sh | 6 ++- scripts/stand-smoke.sh | 7 +++- 8 files changed, 60 insertions(+), 14 deletions(-) diff --git a/compose.yaml b/compose.yaml index 4b7a748..f0ae5d4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -48,6 +48,7 @@ x-airflow-common: &airflow-common AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true" AIRFLOW_ADMIN_USER: ${AIRFLOW_ADMIN_USER:?Скопируйте .env.example в .env} AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?Скопируйте .env.example в .env} + CLICKHOUSE_ETL_PASSWORD: ${CLICKHOUSE_ETL_PASSWORD:?Скопируйте .env.example в .env} volumes: - ./dags:/opt/airflow/dags:ro - ./infra/airflow/init.sh:/opt/airflow/init.sh:ro @@ -86,6 +87,7 @@ x-superset-common: &superset-common SUPERSET__SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://${SUPERSET_METADATA_USER:?Скопируйте .env.example в .env}:${SUPERSET_METADATA_PASSWORD:?Скопируйте .env.example в .env}@postgres-metadata:5432/superset" SUPERSET_ADMIN_USER: ${SUPERSET_ADMIN_USER:?Скопируйте .env.example в .env} SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD:?Скопируйте .env.example в .env} + CLICKHOUSE_BI_PASSWORD: ${CLICKHOUSE_BI_PASSWORD:?Скопируйте .env.example в .env} volumes: - ./infra/superset/superset_config.py:/app/pythonpath/superset_config.py:ro - ./infra/superset/init.sh:/app/infra/init.sh:ro @@ -231,6 +233,7 @@ services: condition: service_healthy environment: LC_ALL: C + CLICKHOUSE_ETL_PASSWORD: ${CLICKHOUSE_ETL_PASSWORD:?Скопируйте .env.example в .env} entrypoint: ["/bin/bash", "-ec"] command: - | @@ -243,6 +246,8 @@ services: printf 'Применяем %s.\n' "$${ddl_file##*/}" clickhouse-client \ --host clickhouse-01 \ + --user etl \ + --password "$$CLICKHOUSE_ETL_PASSWORD" \ --multiquery \ --queries-file "$$ddl_file" done diff --git a/dags/test_clickhouse.py b/dags/test_clickhouse.py index da31590..8fb4f61 100644 --- a/dags/test_clickhouse.py +++ b/dags/test_clickhouse.py @@ -37,10 +37,24 @@ EXPECTED_TABLES = [ # подключение Airflow ходит по HTTP на 8123. NODES = ( ("ноде 1", "system.tables"), - ("ноде 2", "remote('clickhouse-02:9000', system.tables)"), + ( + "ноде 2", + """remote( + 'clickhouse-02:9000', 'system', 'tables', + {remote_user:String}, {remote_password:String} + )""", + ), ) +def _remote_parameters() -> dict[str, str]: + connection = Connection.get("clickhouse_default") + return { + "remote_user": connection.login, + "remote_password": connection.password, + } + + def _clickhouse_client(): # clickhouse_connect импортируется внутри функции, а не наверху файла: # обработчик DAG разбирает этот файл снова и снова, и импорт наверху @@ -52,8 +66,8 @@ def _clickhouse_client(): return clickhouse_connect.get_client( host=connection.host, port=connection.port, - username=connection.login or "default", - password=connection.password or "", + username=connection.login, + password=connection.password, database=connection.schema or "default", connect_timeout=5, send_receive_timeout=30, @@ -68,7 +82,8 @@ def _table_engines(client, source: str) -> list[tuple[str, str]]: WHERE database = 'default' AND name IN ('{LOCAL_TABLE}', '{DISTRIBUTED_TABLE}') ORDER BY name - """ + """, + parameters=_remote_parameters(), ) return result.result_rows @@ -173,8 +188,12 @@ def test_clickhouse(): node_2_rows = client.query( """ SELECT hostName() - FROM remote('clickhouse-02:9000', system.one) - """ + FROM remote( + 'clickhouse-02:9000', 'system', 'one', + {remote_user:String}, {remote_password:String} + ) + """, + parameters=_remote_parameters(), ).result_rows if len(node_2_rows) != 1: raise RuntimeError(f"не удалось определить имя ноды 2: {node_2_rows}") @@ -184,11 +203,16 @@ def test_clickhouse(): FROM remote( 'clickhouse-02:9000', 'default', - '{DISTRIBUTED_TABLE}' + '{DISTRIBUTED_TABLE}', + {{remote_user:String}}, + {{remote_password:String}} ) WHERE marker = {{marker:String}} """, - parameters={"marker": written["marker"]}, + parameters={ + "marker": written["marker"], + **_remote_parameters(), + }, ).result_rows if written["hostname"] == node_2_rows[0][0]: raise RuntimeError( diff --git a/infra/airflow/init.sh b/infra/airflow/init.sh index dbc5d1c..74ec46b 100755 --- a/infra/airflow/init.sh +++ b/infra/airflow/init.sh @@ -26,6 +26,7 @@ runuser -u airflow -- airflow connections add clickhouse_default \ --conn-type generic \ --conn-host clickhouse-01 \ --conn-port 8123 \ - --conn-login default \ + --conn-login etl \ + --conn-password "$CLICKHOUSE_ETL_PASSWORD" \ --conn-schema default \ --conn-description "ClickHouse, нода 1; типизированный провайдер появится на этапе ETL" diff --git a/infra/clickhouse/users.d/access.xml b/infra/clickhouse/users.d/access.xml index b4fc45b..dc45cff 100644 --- a/infra/clickhouse/users.d/access.xml +++ b/infra/clickhouse/users.d/access.xml @@ -14,6 +14,8 @@ GRANT SELECT ON system.* GRANT CLUSTER ON *.* GRANT REMOTE ON *.* + + GRANT KAFKA ON *.* diff --git a/infra/superset/import/databases/clickhouse.yaml b/infra/superset/import/databases/clickhouse.yaml index 4330e58..b52a4a3 100644 --- a/infra/superset/import/databases/clickhouse.yaml +++ b/infra/superset/import/databases/clickhouse.yaml @@ -1,5 +1,5 @@ database_name: ClickHouse -sqlalchemy_uri: clickhousedb://default@clickhouse-02:8123/default +sqlalchemy_uri: clickhousedb://bi@clickhouse-02:8123/default uuid: 4b8f2c6e-1d3a-4f5b-9c7d-2e8a1f0b3c5d cache_timeout: null expose_in_sqllab: true diff --git a/infra/superset/superset_config.py b/infra/superset/superset_config.py index 5ae7413..1ff180f 100644 --- a/infra/superset/superset_config.py +++ b/infra/superset/superset_config.py @@ -5,6 +5,15 @@ import os SQLALCHEMY_DATABASE_URI = os.environ["SUPERSET__SQLALCHEMY_DATABASE_URI"] SECRET_KEY = os.environ["SUPERSET_SECRET_KEY"] + +# Источник данных пока один, поэтому штатному хранилищу паролей Superset +# достаточно вернуть один секрет из окружения. +def clickhouse_password(_url): + return os.environ["CLICKHOUSE_BI_PASSWORD"] + + +SQLALCHEMY_CUSTOM_PASSWORD_STORE = clickhouse_password + SESSION_COOKIE_NAME = "superset_session" SESSION_COOKIE_SECURE = False SESSION_COOKIE_HTTPONLY = True diff --git a/scripts/stand-services.sh b/scripts/stand-services.sh index 9d701d4..205f627 100755 --- a/scripts/stand-services.sh +++ b/scripts/stand-services.sh @@ -237,6 +237,7 @@ check_airflow_probes() { } check_superset() { + local clickhouse_password local config local login local metadata_tables @@ -253,6 +254,7 @@ check_superset() { config="$(compose config --format json 2>/dev/null || true)" user="$(jq -r '.services.superset.environment.SUPERSET_ADMIN_USER // empty' <<<"$config")" password="$(jq -r '.services.superset.environment.SUPERSET_ADMIN_PASSWORD // empty' <<<"$config")" + clickhouse_password="$(jq -r '.services.superset.environment.CLICKHOUSE_BI_PASSWORD // empty' <<<"$config")" superset_user="$(jq -r '.services["postgres-metadata"].environment.SUPERSET_METADATA_USER // empty' <<<"$config")" superset_password="$(jq -r '.services["postgres-metadata"].environment.SUPERSET_METADATA_PASSWORD // empty' <<<"$config")" port="$(published_port superset 8088)" @@ -284,10 +286,10 @@ check_superset() { if [[ "$metadata_tables" == '1' ]] && \ [[ "$metadata_engine" == "postgresql|${superset_user}|postgres-metadata|5432|superset" ]] && \ [[ "$stored_uuid" == '4b8f2c6e-1d3a-4f5b-9c7d-2e8a1f0b3c5d' ]] && \ - [[ "$stored_uri" == 'clickhousedb://default@clickhouse-02:8123/default' ]]; then + [[ "$stored_uri" == "clickhousedb://bi:${clickhouse_password}@clickhouse-02:8123/default" ]]; then pass 'метаданные Superset живут в Postgres, подготовленное подключение указывает на clickhouse-02' else - fail "Superset не подтвердил Postgres и подготовленное подключение к ноде 2: таблицы=${metadata_tables:-нет}, движок=${metadata_engine:-нет}, UUID=${stored_uuid:-нет}, URI=${stored_uri:-нет}" + fail "Superset не подтвердил Postgres и подготовленное подключение bi к ноде 2: таблицы=${metadata_tables:-нет}, движок=${metadata_engine:-нет}, UUID=${stored_uuid:-нет}" return fi diff --git a/scripts/stand-smoke.sh b/scripts/stand-smoke.sh index e6c9595..a29239b 100755 --- a/scripts/stand-smoke.sh +++ b/scripts/stand-smoke.sh @@ -154,6 +154,7 @@ check_grafana_datasource() { check_airflow() { local airflow_port local airflow_token + local clickhouse_password local config local connection local dag @@ -166,6 +167,7 @@ check_airflow() { config="$(compose config --format json 2>/dev/null || true)" user="$(jq -r '.services["airflow-apiserver"].environment.AIRFLOW_ADMIN_USER // empty' <<<"$config")" password="$(jq -r '.services["airflow-apiserver"].environment.AIRFLOW_ADMIN_PASSWORD // empty' <<<"$config")" + clickhouse_password="$(jq -r '.services["airflow-scheduler"].environment.CLICKHOUSE_ETL_PASSWORD // empty' <<<"$config")" airflow_port="$(published_port airflow-apiserver 8080)" health="$(curl -sf --max-time 10 \ @@ -211,12 +213,13 @@ check_airflow() { .connection_id == "clickhouse_default" and .host == "clickhouse-01" and .port == 8123 and - .login == "default" and + .login == "etl" and .schema == "default" ' >/dev/null 2>&1 <<<"$connection" && \ timeout 20s "${COMPOSE_CMD[@]}" --project-directory "$ROOT_DIR" \ exec -T airflow-scheduler \ - curl -sf 'http://clickhouse-01:8123/?query=SELECT%201' \ + curl -sf -u "etl:${clickhouse_password}" \ + 'http://clickhouse-01:8123/?query=SELECT%201' \ 2>/dev/null | grep -qx '1'; then pass 'подготовленное подключение Airflow указывает на clickhouse-01, нода доступна из контейнера' else