diff --git a/.gitignore b/.gitignore index d4c6094..7775eb3 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,4 @@ $RECYCLE.BIN/ *.msm *.msp *.lnk +node_modules diff --git a/README.md b/README.md index 99cbd7c..cd30275 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ curl http://localhost:8088/health # должно вернуть 200 ``` Что создаётся автоматически: -- **Подключение к ClickHouse**: `clickhouse_dwh` (URI: `clickhouse+connect://default@clickhouse:8123/default`) +- **Подключение к ClickHouse**: `clickhouse_dwh` (URI: `clickhousedb://default:123456@clickhouse:8123/default`) - **Датасеты** (6 шт.): `v_events_enriched`, `v_daily_traffic`, `v_utm_effectiveness`, `v_top_pages_daily`, `v_session_overview`, `dq_summary` - **Чарты** (10 шт.): KPI метрики, графики трафика, география, UTM-эффективность, качество данных - **Дашборд**: "🛒 E-commerce Analytics Dashboard" diff --git a/docker-compose.yml b/docker-compose.yml index 1232b7d..7054a79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -273,7 +273,9 @@ services: echo 'Initializing roles...' && superset init && echo 'Creating ClickHouse connection and datasets...' && - python /app/superset_init/init_superset.py || true && + python /app/superset_init/init_superset.py && + echo 'Creating dashboard and charts...' && + python /app/superset_init/create_dashboard.py && echo 'Superset initialized successfully' " depends_on: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 56b7362..4630e71 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -602,7 +602,7 @@ INSERT INTO dm.daily_traffic SELECT * FROM dm.v_daily_traffic; - URL: `clickhouse://default:123456@clickhouse:9000/default` (native TCP для Airflow plugin) - Provider/интеграция: `airflow-clickhouse-plugin` (в `airflow/requirements.txt`), задачи выполняются через `ClickHouseOperator`. - Дополнительно: `kafka-python==2.0.6` для работы с Kafka из DAG. -- Примечание: Superset подключается к ClickHouse по HTTP (обычно `clickhouse+connect://...:8123/...`). +- Примечание: Superset подключается к ClickHouse по HTTP (обычно `clickhousedb://...:8123/...`). --- diff --git a/docs/SUPERSET_DASHBOARD.md b/docs/SUPERSET_DASHBOARD.md index 6248fa8..2fe6a04 100644 --- a/docs/SUPERSET_DASHBOARD.md +++ b/docs/SUPERSET_DASHBOARD.md @@ -93,7 +93,7 @@ make superset-dashboard # Основные make up # Запуск всех сервисов make down # Остановка сервисов -make down-v # Остановка с удалением volumes +make clean # Остановка с удалением volumes make logs service=superset # Логи сервиса # ETL @@ -120,7 +120,7 @@ make superset-restart # Перезапуск сервиса 3. Выберите **ClickHouse** 4. Введите SQLAlchemy URI: ``` - clickhouse+native://default@clickhouse:9000/default + clickhousedb://default:123456@clickhouse:8123/default ``` 5. Установите: - **Expose in SQL Lab:** ✅ diff --git a/superset/dashboards/ecommerce_analytics.zip.json b/superset/dashboards/ecommerce_analytics.zip.json index 76d3e90..ca643c0 100644 --- a/superset/dashboards/ecommerce_analytics.zip.json +++ b/superset/dashboards/ecommerce_analytics.zip.json @@ -188,7 +188,7 @@ { "__Database__": { "database_name": "clickhouse_dwh", - "sqlalchemy_uri": "clickhouse+native://default@clickhouse:9000/default", + "sqlalchemy_uri": "clickhousedb://default:123456@clickhouse:8123/default", "expose_in_sqllab": true, "allow_ctas": false, "allow_cvas": false, diff --git a/superset/init_superset.py b/superset/init_superset.py index 2afbb22..d4eb1c3 100644 --- a/superset/init_superset.py +++ b/superset/init_superset.py @@ -24,12 +24,11 @@ ================================================================================ """ -import os import sys -import json +import os import logging import subprocess -from typing import Optional +from urllib.parse import quote_plus # Настройка логирования logging.basicConfig( @@ -41,6 +40,22 @@ logger = logging.getLogger(__name__) # Добавляем путь к superset sys.path.insert(0, '/app') +CLICKHOUSE_USER = os.getenv('CLICKHOUSE_USER', 'default') +CLICKHOUSE_PASSWORD = os.getenv('CLICKHOUSE_PASSWORD', '123456') +CLICKHOUSE_HOST = os.getenv('CLICKHOUSE_HOST', 'clickhouse') +CLICKHOUSE_PORT = os.getenv('CLICKHOUSE_HTTP_PORT', '8123') +CLICKHOUSE_DATABASE = os.getenv('CLICKHOUSE_DATABASE', 'default') + + +def build_clickhouse_uri() -> str: + """Собирает URI подключения к ClickHouse для Superset.""" + user = quote_plus(CLICKHOUSE_USER) + password = quote_plus(CLICKHOUSE_PASSWORD) + host = CLICKHOUSE_HOST + port = CLICKHOUSE_PORT + database = quote_plus(CLICKHOUSE_DATABASE) + return f"clickhousedb://{user}:{password}@{host}:{port}/{database}" + def run_superset_cli(args): """Запуск команды superset CLI""" @@ -57,13 +72,14 @@ def run_superset_cli(args): def create_clickhouse_connection(): """Создание подключения к ClickHouse через CLI""" logger.info("Creating ClickHouse database connection...") + database_uri = build_clickhouse_uri() # Создаем подключение через set-database-uri # clickhouse-connect использует HTTP порт 8123 внутри Docker сети success, stdout, stderr = run_superset_cli([ 'set-database-uri', '-d', 'clickhouse_dwh', - '-u', 'clickhouse+connect://default@clickhouse:8123/default' + '-u', database_uri ]) if success: @@ -74,7 +90,7 @@ def create_clickhouse_connection(): logger.info("Please create manually via UI:") logger.info("1. Go to: Settings → Database Connections → + Database") logger.info("2. Select: ClickHouse") - logger.info("3. URI: clickhouse+connect://default@clickhouse:8123/default") + logger.info(f"3. URI: {database_uri}") return False @@ -136,31 +152,65 @@ def import_datasets(): "print(f'Found database: {database.database_name} (id={database.id})')", "", "imported = 0", + "refreshed = 0", + "errors = 0", ] for ds in datasets: + table_name = ds["table_name"] + schema_name = ds["schema"] + description = ds["description"] script_lines.extend([ "", - f"# Dataset: {ds['table_name']}", - f"existing = db.session.query(SqlaTable).filter_by(table_name='{ds['table_name']}', schema='{ds['schema']}').first()", + f"# Dataset: {table_name}", + ( + "existing = db.session.query(SqlaTable).filter_by(" + f"table_name={table_name!r}, schema={schema_name!r}" + ").first()" + ), "if existing:", - f" print(f'Dataset {ds['table_name']} already exists')", + " try:", + " if not existing.columns:", + " existing.fetch_metadata()", + " db.session.commit()", + f" print('Refreshed dataset metadata: {table_name}')", + " refreshed += 1", + " else:", + f" print('Dataset {table_name} already exists')", + " except Exception as e:", + f" print(f'ERROR: failed to refresh {table_name}: {{e}}')", + " db.session.rollback()", + " errors += 1", "else:", " try:", - f" dataset = SqlaTable(table_name='{ds['table_name']}', schema='{ds['schema']}', database_id=database.id, database=database, description='{ds['description']}')", + ( + " dataset = SqlaTable(" + f"table_name={table_name!r}, " + f"schema={schema_name!r}, " + "database_id=database.id, " + "database=database, " + f"description={description!r}" + ")" + ), " db.session.add(dataset)", " db.session.flush()", - f" print(f'Created dataset: {ds['table_name']}')", + " dataset.fetch_metadata()", + " db.session.commit()", + f" print('Created dataset: {table_name}')", " imported += 1", " except Exception as e:", - f" print(f'Error creating {ds['table_name']}: {{e}}')", + f" print(f'ERROR: failed to create {table_name}: {{e}}')", " db.session.rollback()", + " errors += 1", ]) script_lines.extend([ "", - "db.session.commit()", "print(f'Successfully imported {imported} datasets')", + "print(f'Refreshed metadata for {refreshed} datasets')", + "print(f'Errors: {errors}')", + "if errors > 0:", + " raise SystemExit(1)", ]) script_content = '\n'.join(script_lines)