diff --git a/Dockerfile.superset b/Dockerfile.superset index e8608b3..2dfd337 100644 --- a/Dockerfile.superset +++ b/Dockerfile.superset @@ -6,6 +6,6 @@ RUN apt-get update && \ apt-get install -y bind9-host bat less iputils-ping curl postgresql-client && \ apt-get clean -USER superset +RUN pip install --no-cache-dir clickhouse-connect==0.8.* psycopg2-binary==2.9.* -RUN pip install clickhouse-connect==0.8.* clickhouse-sqlalchemy==0.3.* psycopg2-binary==2.9.* +USER superset diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index ac69d8c..7ee8cc6 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -293,5 +293,13 @@ curl -s -X POST -u admin:admin http://localhost:3000/api/admin/provisioning/aler docker volume rm _grafana_lib docker compose up -d grafana ``` +- В Superset ошибки `DB engine Error` и `Cannot load filter`, а в логах есть `Can't load plugin: sqlalchemy.dialects:clickhouse.connect`: + - Причина: некорректный URI диалекта ClickHouse (`clickhouse+connect://...`). + - Используйте URI `clickhousedb://...` и пересоберите сервисы Superset: + ```bash + docker compose build superset superset-init + docker compose up -d clickhouse + docker compose up -d --force-recreate superset-init superset + ``` - После `docker compose down -v` нужно повторно прогнать: `ddl_init` -> `kafka_load` -> `etl_pipeline`. - Для демо по умолчанию использовать малый срез данных; полный прогон делать осознанно. diff --git a/superset/init_superset.py b/superset/init_superset.py index e3946cc..679eb8b 100644 --- a/superset/init_superset.py +++ b/superset/init_superset.py @@ -54,7 +54,7 @@ def build_clickhouse_uri() -> str: host = CLICKHOUSE_HOST port = CLICKHOUSE_PORT database = quote_plus(CLICKHOUSE_DATABASE) - return f"clickhouse+connect://{user}:{password}@{host}:{port}/{database}" + return f"clickhousedb://{user}:{password}@{host}:{port}/{database}" def run_superset_cli(args): @@ -139,7 +139,7 @@ def import_datasets(): # Создаем Python скрипт для выполнения внутри superset shell script_lines = [ - "import clickhouse_connect # Регистрирует диалект clickhouse+connect", + "import clickhouse_connect # Регистрирует диалект clickhousedb", "from superset.extensions import db", "from superset.models.core import Database", "from superset.connectors.sqla.models import SqlaTable", @@ -170,7 +170,18 @@ def import_datasets(): ").first()" ), "if existing:", - f" print('Dataset {table_name} already exists')", + " try:", + " if not existing.columns:", + " existing.fetch_metadata()", + " db.session.commit()", + f" print('Refreshed dataset metadata: {table_name}')", + " refreshed += 1", + " else:", + f" print('Dataset {table_name} already exists')", + " except Exception as e:", + f" print(f'ERROR: failed to refresh {table_name}: {{e}}')", + " db.session.rollback()", + " errors += 1", "else:", " try:", ( @@ -183,8 +194,10 @@ def import_datasets(): ")" ), " db.session.add(dataset)", + " db.session.flush()", + " dataset.fetch_metadata()", " db.session.commit()", - f" print('Created dataset: {table_name} (metadata will be fetched on first use)')", + f" print('Created dataset: {table_name}')", " imported += 1", " except Exception as e:", f" print(f'ERROR: failed to create {table_name}: {{e}}')",