diff --git a/compose.yaml b/compose.yaml
index 4b7a748..f0ae5d4 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -48,6 +48,7 @@ x-airflow-common: &airflow-common
AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true"
AIRFLOW_ADMIN_USER: ${AIRFLOW_ADMIN_USER:?Скопируйте .env.example в .env}
AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:?Скопируйте .env.example в .env}
+ CLICKHOUSE_ETL_PASSWORD: ${CLICKHOUSE_ETL_PASSWORD:?Скопируйте .env.example в .env}
volumes:
- ./dags:/opt/airflow/dags:ro
- ./infra/airflow/init.sh:/opt/airflow/init.sh:ro
@@ -86,6 +87,7 @@ x-superset-common: &superset-common
SUPERSET__SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://${SUPERSET_METADATA_USER:?Скопируйте .env.example в .env}:${SUPERSET_METADATA_PASSWORD:?Скопируйте .env.example в .env}@postgres-metadata:5432/superset"
SUPERSET_ADMIN_USER: ${SUPERSET_ADMIN_USER:?Скопируйте .env.example в .env}
SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD:?Скопируйте .env.example в .env}
+ CLICKHOUSE_BI_PASSWORD: ${CLICKHOUSE_BI_PASSWORD:?Скопируйте .env.example в .env}
volumes:
- ./infra/superset/superset_config.py:/app/pythonpath/superset_config.py:ro
- ./infra/superset/init.sh:/app/infra/init.sh:ro
@@ -231,6 +233,7 @@ services:
condition: service_healthy
environment:
LC_ALL: C
+ CLICKHOUSE_ETL_PASSWORD: ${CLICKHOUSE_ETL_PASSWORD:?Скопируйте .env.example в .env}
entrypoint: ["/bin/bash", "-ec"]
command:
- |
@@ -243,6 +246,8 @@ services:
printf 'Применяем %s.\n' "$${ddl_file##*/}"
clickhouse-client \
--host clickhouse-01 \
+ --user etl \
+ --password "$$CLICKHOUSE_ETL_PASSWORD" \
--multiquery \
--queries-file "$$ddl_file"
done
diff --git a/dags/test_clickhouse.py b/dags/test_clickhouse.py
index da31590..8fb4f61 100644
--- a/dags/test_clickhouse.py
+++ b/dags/test_clickhouse.py
@@ -37,10 +37,24 @@ EXPECTED_TABLES = [
# подключение Airflow ходит по HTTP на 8123.
NODES = (
("ноде 1", "system.tables"),
- ("ноде 2", "remote('clickhouse-02:9000', system.tables)"),
+ (
+ "ноде 2",
+ """remote(
+ 'clickhouse-02:9000', 'system', 'tables',
+ {remote_user:String}, {remote_password:String}
+ )""",
+ ),
)
+def _remote_parameters() -> dict[str, str]:
+ connection = Connection.get("clickhouse_default")
+ return {
+ "remote_user": connection.login,
+ "remote_password": connection.password,
+ }
+
+
def _clickhouse_client():
# clickhouse_connect импортируется внутри функции, а не наверху файла:
# обработчик DAG разбирает этот файл снова и снова, и импорт наверху
@@ -52,8 +66,8 @@ def _clickhouse_client():
return clickhouse_connect.get_client(
host=connection.host,
port=connection.port,
- username=connection.login or "default",
- password=connection.password or "",
+ username=connection.login,
+ password=connection.password,
database=connection.schema or "default",
connect_timeout=5,
send_receive_timeout=30,
@@ -68,7 +82,8 @@ def _table_engines(client, source: str) -> list[tuple[str, str]]:
WHERE database = 'default'
AND name IN ('{LOCAL_TABLE}', '{DISTRIBUTED_TABLE}')
ORDER BY name
- """
+ """,
+ parameters=_remote_parameters(),
)
return result.result_rows
@@ -173,8 +188,12 @@ def test_clickhouse():
node_2_rows = client.query(
"""
SELECT hostName()
- FROM remote('clickhouse-02:9000', system.one)
- """
+ FROM remote(
+ 'clickhouse-02:9000', 'system', 'one',
+ {remote_user:String}, {remote_password:String}
+ )
+ """,
+ parameters=_remote_parameters(),
).result_rows
if len(node_2_rows) != 1:
raise RuntimeError(f"не удалось определить имя ноды 2: {node_2_rows}")
@@ -184,11 +203,16 @@ def test_clickhouse():
FROM remote(
'clickhouse-02:9000',
'default',
- '{DISTRIBUTED_TABLE}'
+ '{DISTRIBUTED_TABLE}',
+ {{remote_user:String}},
+ {{remote_password:String}}
)
WHERE marker = {{marker:String}}
""",
- parameters={"marker": written["marker"]},
+ parameters={
+ "marker": written["marker"],
+ **_remote_parameters(),
+ },
).result_rows
if written["hostname"] == node_2_rows[0][0]:
raise RuntimeError(
diff --git a/infra/airflow/init.sh b/infra/airflow/init.sh
index dbc5d1c..74ec46b 100755
--- a/infra/airflow/init.sh
+++ b/infra/airflow/init.sh
@@ -26,6 +26,7 @@ runuser -u airflow -- airflow connections add clickhouse_default \
--conn-type generic \
--conn-host clickhouse-01 \
--conn-port 8123 \
- --conn-login default \
+ --conn-login etl \
+ --conn-password "$CLICKHOUSE_ETL_PASSWORD" \
--conn-schema default \
--conn-description "ClickHouse, нода 1; типизированный провайдер появится на этапе ETL"
diff --git a/infra/clickhouse/users.d/access.xml b/infra/clickhouse/users.d/access.xml
index b4fc45b..dc45cff 100644
--- a/infra/clickhouse/users.d/access.xml
+++ b/infra/clickhouse/users.d/access.xml
@@ -14,6 +14,8 @@
GRANT SELECT ON system.*
GRANT CLUSTER ON *.*
GRANT REMOTE ON *.*
+
+ GRANT KAFKA ON *.*
diff --git a/infra/superset/import/databases/clickhouse.yaml b/infra/superset/import/databases/clickhouse.yaml
index 4330e58..b52a4a3 100644
--- a/infra/superset/import/databases/clickhouse.yaml
+++ b/infra/superset/import/databases/clickhouse.yaml
@@ -1,5 +1,5 @@
database_name: ClickHouse
-sqlalchemy_uri: clickhousedb://default@clickhouse-02:8123/default
+sqlalchemy_uri: clickhousedb://bi@clickhouse-02:8123/default
uuid: 4b8f2c6e-1d3a-4f5b-9c7d-2e8a1f0b3c5d
cache_timeout: null
expose_in_sqllab: true
diff --git a/infra/superset/superset_config.py b/infra/superset/superset_config.py
index 5ae7413..1ff180f 100644
--- a/infra/superset/superset_config.py
+++ b/infra/superset/superset_config.py
@@ -5,6 +5,15 @@ import os
SQLALCHEMY_DATABASE_URI = os.environ["SUPERSET__SQLALCHEMY_DATABASE_URI"]
SECRET_KEY = os.environ["SUPERSET_SECRET_KEY"]
+
+# Источник данных пока один, поэтому штатному хранилищу паролей Superset
+# достаточно вернуть один секрет из окружения.
+def clickhouse_password(_url):
+ return os.environ["CLICKHOUSE_BI_PASSWORD"]
+
+
+SQLALCHEMY_CUSTOM_PASSWORD_STORE = clickhouse_password
+
SESSION_COOKIE_NAME = "superset_session"
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_HTTPONLY = True
diff --git a/scripts/stand-services.sh b/scripts/stand-services.sh
index 9d701d4..205f627 100755
--- a/scripts/stand-services.sh
+++ b/scripts/stand-services.sh
@@ -237,6 +237,7 @@ check_airflow_probes() {
}
check_superset() {
+ local clickhouse_password
local config
local login
local metadata_tables
@@ -253,6 +254,7 @@ check_superset() {
config="$(compose config --format json 2>/dev/null || true)"
user="$(jq -r '.services.superset.environment.SUPERSET_ADMIN_USER // empty' <<<"$config")"
password="$(jq -r '.services.superset.environment.SUPERSET_ADMIN_PASSWORD // empty' <<<"$config")"
+ clickhouse_password="$(jq -r '.services.superset.environment.CLICKHOUSE_BI_PASSWORD // empty' <<<"$config")"
superset_user="$(jq -r '.services["postgres-metadata"].environment.SUPERSET_METADATA_USER // empty' <<<"$config")"
superset_password="$(jq -r '.services["postgres-metadata"].environment.SUPERSET_METADATA_PASSWORD // empty' <<<"$config")"
port="$(published_port superset 8088)"
@@ -284,10 +286,10 @@ check_superset() {
if [[ "$metadata_tables" == '1' ]] && \
[[ "$metadata_engine" == "postgresql|${superset_user}|postgres-metadata|5432|superset" ]] && \
[[ "$stored_uuid" == '4b8f2c6e-1d3a-4f5b-9c7d-2e8a1f0b3c5d' ]] && \
- [[ "$stored_uri" == 'clickhousedb://default@clickhouse-02:8123/default' ]]; then
+ [[ "$stored_uri" == "clickhousedb://bi:${clickhouse_password}@clickhouse-02:8123/default" ]]; then
pass 'метаданные Superset живут в Postgres, подготовленное подключение указывает на clickhouse-02'
else
- fail "Superset не подтвердил Postgres и подготовленное подключение к ноде 2: таблицы=${metadata_tables:-нет}, движок=${metadata_engine:-нет}, UUID=${stored_uuid:-нет}, URI=${stored_uri:-нет}"
+ fail "Superset не подтвердил Postgres и подготовленное подключение bi к ноде 2: таблицы=${metadata_tables:-нет}, движок=${metadata_engine:-нет}, UUID=${stored_uuid:-нет}"
return
fi
diff --git a/scripts/stand-smoke.sh b/scripts/stand-smoke.sh
index e6c9595..a29239b 100755
--- a/scripts/stand-smoke.sh
+++ b/scripts/stand-smoke.sh
@@ -154,6 +154,7 @@ check_grafana_datasource() {
check_airflow() {
local airflow_port
local airflow_token
+ local clickhouse_password
local config
local connection
local dag
@@ -166,6 +167,7 @@ check_airflow() {
config="$(compose config --format json 2>/dev/null || true)"
user="$(jq -r '.services["airflow-apiserver"].environment.AIRFLOW_ADMIN_USER // empty' <<<"$config")"
password="$(jq -r '.services["airflow-apiserver"].environment.AIRFLOW_ADMIN_PASSWORD // empty' <<<"$config")"
+ clickhouse_password="$(jq -r '.services["airflow-scheduler"].environment.CLICKHOUSE_ETL_PASSWORD // empty' <<<"$config")"
airflow_port="$(published_port airflow-apiserver 8080)"
health="$(curl -sf --max-time 10 \
@@ -211,12 +213,13 @@ check_airflow() {
.connection_id == "clickhouse_default" and
.host == "clickhouse-01" and
.port == 8123 and
- .login == "default" and
+ .login == "etl" and
.schema == "default"
' >/dev/null 2>&1 <<<"$connection" && \
timeout 20s "${COMPOSE_CMD[@]}" --project-directory "$ROOT_DIR" \
exec -T airflow-scheduler \
- curl -sf 'http://clickhouse-01:8123/?query=SELECT%201' \
+ curl -sf -u "etl:${clickhouse_password}" \
+ 'http://clickhouse-01:8123/?query=SELECT%201' \
2>/dev/null | grep -qx '1'; then
pass 'подготовленное подключение Airflow указывает на clickhouse-01, нода доступна из контейнера'
else