feat: добавлен дашборд Superset для e-commerce аналитики
- Добавлен сервис superset-init в docker-compose для автоматической инициализации - Созданы Python-скрипты для инициализации подключения ClickHouse и создания датасетов - Создан скрипт для автоматического создания дашборда с 10 чартами - Создан скрипт экспорта дашборда в JSON - Добавлен экспортируемый JSON дашборда (ecommerce_analytics.zip.json) - Обновлен Makefile с командами superset-init, superset-dashboard, superset-export - Добавлена документация docs/SUPERSET_DASHBOARD.md Дашборд включает: - KPI блок (Total Events, Unique Users, Sessions, Avg/Session) - Динамика трафика (Events by Hour, Traffic by Device) - География (World Map) - Маркетинг (UTM Effectiveness Table, Top Pages) - Качество данных (DQ Summary) - Native Filters (Date Range, Country, Device, Browser)
This commit is contained in:
@@ -1,7 +1,13 @@
|
|||||||
.PHONY: up down clean ddl data transform reload-monitoring recover-monitoring
|
.PHONY: up down clean ddl data transform logs \
|
||||||
|
reload-monitoring recover-monitoring \
|
||||||
|
superset-init superset-dashboard superset-export superset-ui superset-restart
|
||||||
|
|
||||||
COMPOSE ?= docker compose
|
COMPOSE ?= docker compose
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Основные команды
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
up:
|
up:
|
||||||
$(COMPOSE) up -d
|
$(COMPOSE) up -d
|
||||||
|
|
||||||
@@ -13,6 +19,13 @@ down:
|
|||||||
clean:
|
clean:
|
||||||
$(COMPOSE) down -v --remove-orphans
|
$(COMPOSE) down -v --remove-orphans
|
||||||
|
|
||||||
|
logs:
|
||||||
|
$(COMPOSE) logs -f --tail=200 $(service)
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# ETL Pipeline
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
ddl:
|
ddl:
|
||||||
bash ./scripts/apply_clickhouse_ddl.sh
|
bash ./scripts/apply_clickhouse_ddl.sh
|
||||||
|
|
||||||
@@ -49,3 +62,28 @@ recover-monitoring:
|
|||||||
@echo "=== Проверка targets ==="
|
@echo "=== Проверка targets ==="
|
||||||
@curl -s http://localhost:9090/api/v1/targets | grep -o '"job":"[^"]*"' | sort | uniq
|
@curl -s http://localhost:9090/api/v1/targets | grep -o '"job":"[^"]*"' | sort | uniq
|
||||||
@curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"' | sort | uniq -c
|
@curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"[^"]*"' | sort | uniq -c
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Superset команды
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Инициализация Superset (подключение к ClickHouse + датасеты)
|
||||||
|
superset-init:
|
||||||
|
$(COMPOSE) exec -T superset bash -c "python /app/superset_init/init_superset.py"
|
||||||
|
|
||||||
|
# Создание дашборда с чартами
|
||||||
|
superset-dashboard:
|
||||||
|
$(COMPOSE) exec -T superset bash -c "python /app/superset_init/create_dashboard.py"
|
||||||
|
|
||||||
|
# Экспорт дашборда в JSON
|
||||||
|
superset-export:
|
||||||
|
$(COMPOSE) exec -T superset bash -c "python /app/superset_init/export_dashboard.py"
|
||||||
|
|
||||||
|
# Открыть Superset UI
|
||||||
|
superset-ui:
|
||||||
|
@echo "Superset доступен по адресу: http://localhost:8088"
|
||||||
|
@echo "Логин: admin / Пароль: admin"
|
||||||
|
|
||||||
|
# Перезапуск Superset
|
||||||
|
superset-restart:
|
||||||
|
$(COMPOSE) restart superset
|
||||||
|
|||||||
@@ -227,14 +227,63 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- superset_data:/var/lib/superset
|
- superset_data:/var/lib/superset
|
||||||
- superset_config:/app/superset_home
|
- superset_config:/app/superset_home
|
||||||
|
- ./superset:/app/superset_init:ro
|
||||||
environment:
|
environment:
|
||||||
- SUPERSET_SECRET_KEY=9wc5+erMt60+lxrXDf3RjeIR+zONpEFusO00Np7JzfliMTI1e+RXnHcQ
|
- SUPERSET_SECRET_KEY=9wc5+erMt60+lxrXDf3RjeIR+zONpEFusO00Np7JzfliMTI1e+RXnHcQ
|
||||||
- TZ=Europe/Moscow
|
- TZ=Europe/Moscow
|
||||||
|
- DATABASE_DB=superset
|
||||||
|
- DATABASE_HOST=postgres-metadata
|
||||||
|
- DATABASE_PASSWORD=airflow
|
||||||
|
- DATABASE_USER=airflow
|
||||||
|
- DATABASE_PORT=5432
|
||||||
|
- DATABASE_DIALECT=postgresql
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
|
test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
depends_on:
|
||||||
|
superset-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
|
||||||
|
# Superset initialization service
|
||||||
|
superset-init:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile.superset
|
||||||
|
user: "0:0"
|
||||||
|
networks:
|
||||||
|
- cs_dwh
|
||||||
|
volumes:
|
||||||
|
- superset_data:/var/lib/superset
|
||||||
|
- superset_config:/app/superset_home
|
||||||
|
- ./superset:/app/superset_init:ro
|
||||||
|
environment:
|
||||||
|
- SUPERSET_SECRET_KEY=9wc5+erMt60+lxrXDf3RjeIR+zONpEFusO00Np7JzfliMTI1e+RXnHcQ
|
||||||
|
- TZ=Europe/Moscow
|
||||||
|
- DATABASE_DB=superset
|
||||||
|
- DATABASE_HOST=postgres-metadata
|
||||||
|
- DATABASE_PASSWORD=airflow
|
||||||
|
- DATABASE_USER=airflow
|
||||||
|
- DATABASE_PORT=5432
|
||||||
|
- DATABASE_DIALECT=postgresql
|
||||||
|
command: >
|
||||||
|
bash -ceuo pipefail "
|
||||||
|
echo 'Waiting for PostgreSQL...' &&
|
||||||
|
sleep 10 &&
|
||||||
|
echo 'Initializing Superset DB...' &&
|
||||||
|
superset db upgrade &&
|
||||||
|
echo 'Creating admin user...' &&
|
||||||
|
superset fab create-admin --username admin --password admin --firstname Superset --lastname Admin --email admin@example.org || true &&
|
||||||
|
echo 'Initializing roles...' &&
|
||||||
|
superset init &&
|
||||||
|
echo 'Creating ClickHouse connection and datasets...' &&
|
||||||
|
python /app/superset_init/init_superset.py || true &&
|
||||||
|
echo 'Superset initialized successfully'
|
||||||
|
"
|
||||||
|
depends_on:
|
||||||
|
postgres-metadata:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
# Kafka Exporter для мониторинга через Prometheus
|
# Kafka Exporter для мониторинга через Prometheus
|
||||||
kafka-exporter:
|
kafka-exporter:
|
||||||
|
|||||||
@@ -0,0 +1,262 @@
|
|||||||
|
# Дашборд Superset для E-commerce Analytics
|
||||||
|
|
||||||
|
Документация по настройке и использованию Superset дашборда для анализа кликстрима.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Быстрый старт
|
||||||
|
|
||||||
|
### 1. Запуск инфраструктуры
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Запуск всех сервисов
|
||||||
|
make up
|
||||||
|
|
||||||
|
# Применение DDL в ClickHouse
|
||||||
|
make ddl
|
||||||
|
|
||||||
|
# Загрузка данных в Kafka
|
||||||
|
make data
|
||||||
|
|
||||||
|
# Запуск ETL-пайплайна (ODS → DDS → DM)
|
||||||
|
make transform
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Инициализация Superset
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Автоматическая инициализация (создание подключения и датасетов)
|
||||||
|
make superset-init
|
||||||
|
|
||||||
|
# Создание дашборда с чартами
|
||||||
|
make superset-dashboard
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Доступ к UI
|
||||||
|
|
||||||
|
Откройте в браузере: http://localhost:8088
|
||||||
|
|
||||||
|
**Логин:** `admin`
|
||||||
|
**Пароль:** `admin`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Структура дашборда
|
||||||
|
|
||||||
|
### Витрины данных (Datasets)
|
||||||
|
|
||||||
|
| Витрина | Таблица ClickHouse | Описание |
|
||||||
|
|---------|-------------------|------------|
|
||||||
|
| **Events Enriched** | `dm.v_events_enriched` | Полная обогащённая витрина событий |
|
||||||
|
| **Daily Traffic** | `dm.v_daily_traffic` | Агрегаты по дням |
|
||||||
|
| **UTM Effectiveness** | `dm.v_utm_effectiveness` | Эффективность маркетинговых каналов |
|
||||||
|
| **Top Pages** | `dm.v_top_pages_daily` | Популярность страниц |
|
||||||
|
| **Session Overview** | `dm.v_session_overview` | Анализ сессий |
|
||||||
|
| **DQ Summary** | `dm.dq_summary` | Качество данных |
|
||||||
|
|
||||||
|
### Чарты (Charts)
|
||||||
|
|
||||||
|
#### KPI-блок (верх дашборда)
|
||||||
|
- **📊 Total Events** — общее количество событий
|
||||||
|
- **👤 Unique Users** — уникальные пользователи
|
||||||
|
- **🎯 Unique Sessions** — уникальные сессии (click_id)
|
||||||
|
- **📈 Avg Events/Session** — среднее количество событий на сессию
|
||||||
|
|
||||||
|
#### Динамика трафика
|
||||||
|
- **📅 Events by Hour** — линейный график событий по часам
|
||||||
|
- **📱 Traffic by Device** — pie chart распределения по устройствам
|
||||||
|
|
||||||
|
#### География
|
||||||
|
- **🌍 Geography Map** — world map с распределением по странам
|
||||||
|
|
||||||
|
#### Маркетинг
|
||||||
|
- **🔗 UTM Effectiveness Table** — таблица эффективности UTM-меток
|
||||||
|
- **📄 Top Pages** — bar chart топ-20 страниц
|
||||||
|
|
||||||
|
#### Качество данных
|
||||||
|
- **🔍 Data Quality Summary** — статистика по слоям STG/ODS/DDS
|
||||||
|
|
||||||
|
### Фильтры (Native Filters)
|
||||||
|
|
||||||
|
| Фильтр | Поле | Тип | Применение |
|
||||||
|
|--------|------|-----|------------|
|
||||||
|
| 📅 Date Range | `event_date` | Time Range | Все чарты |
|
||||||
|
| 🌍 Country | `geo_country` | Multi-select | Все чарты |
|
||||||
|
| 📱 Device Type | `device_type` | Multi-select | Все чарты |
|
||||||
|
| 🌐 Browser | `browser_name` | Multi-select | Все чарты |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Команды Makefile
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Основные
|
||||||
|
make up # Запуск всех сервисов
|
||||||
|
make down # Остановка сервисов
|
||||||
|
make down-v # Остановка с удалением volumes
|
||||||
|
make logs service=superset # Логи сервиса
|
||||||
|
|
||||||
|
# ETL
|
||||||
|
make ddl # Применение DDL в ClickHouse
|
||||||
|
make data # Загрузка данных в Kafka
|
||||||
|
make transform # Запуск batch-процесса
|
||||||
|
|
||||||
|
# Superset
|
||||||
|
make superset-init # Инициализация (подключение + датасеты)
|
||||||
|
make superset-dashboard # Создание дашборда
|
||||||
|
make superset-export # Экспорт дашборда в JSON
|
||||||
|
make superset-ui # Показать URL и логин
|
||||||
|
make superset-restart # Перезапуск сервиса
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Ручная настройка (если автоматика не сработала)
|
||||||
|
|
||||||
|
### Создание подключения к ClickHouse
|
||||||
|
|
||||||
|
1. Откройте **Settings → Database Connections**
|
||||||
|
2. Нажмите **+ Database**
|
||||||
|
3. Выберите **ClickHouse**
|
||||||
|
4. Введите SQLAlchemy URI:
|
||||||
|
```
|
||||||
|
clickhouse+native://default@clickhouse:9000/default
|
||||||
|
```
|
||||||
|
5. Установите:
|
||||||
|
- **Expose in SQL Lab:** ✅
|
||||||
|
- **Allow DDL:** ❌
|
||||||
|
6. Нажмите **Connect**
|
||||||
|
|
||||||
|
### Импорт датасетов
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Внутри контейнера
|
||||||
|
docker compose exec superset bash
|
||||||
|
python /app/superset_init/init_superset.py
|
||||||
|
```
|
||||||
|
|
||||||
|
### Создание чартов вручную
|
||||||
|
|
||||||
|
1. Перейдите в **Charts → + Chart**
|
||||||
|
2. Выберите датасет (например, `dm.v_events_enriched`)
|
||||||
|
3. Настройте визуализацию:
|
||||||
|
- **Viz Type:** Big Number / Line Chart / Pie Chart / World Map / Table
|
||||||
|
- **Metrics:** COUNT(*), COUNT(DISTINCT ...)
|
||||||
|
- **Dimensions:** группировки
|
||||||
|
- **Filters:** фильтры
|
||||||
|
4. Нажмите **Create Chart**
|
||||||
|
|
||||||
|
### Создание дашборда
|
||||||
|
|
||||||
|
1. **Dashboards → + Dashboard**
|
||||||
|
2. Назовите: "E-commerce Analytics Dashboard"
|
||||||
|
3. Добавьте чарты из списка
|
||||||
|
4. Настройте layout (drag-and-drop)
|
||||||
|
5. Добавьте Native Filters (фильтры вверху)
|
||||||
|
6. Сохраните
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Экспорт и импорт дашборда
|
||||||
|
|
||||||
|
### Экспорт
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Автоматический экспорт в JSON
|
||||||
|
make superset-export
|
||||||
|
|
||||||
|
# Результат: superset/dashboards/ecommerce_analytics.json
|
||||||
|
```
|
||||||
|
|
||||||
|
### Импорт
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Импорт через CLI
|
||||||
|
docker compose exec superset superset import-dashboards -p /app/superset_init/dashboards/ecommerce_analytics.json
|
||||||
|
|
||||||
|
# Или через UI: Settings → Import Dashboards
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Расширение дашборда
|
||||||
|
|
||||||
|
### Добавление нового чарта
|
||||||
|
|
||||||
|
1. Отредактируйте `superset/create_dashboard.py`
|
||||||
|
2. Добавьте конфигурацию в `CHARTS_CONFIG`
|
||||||
|
3. Запустите: `make superset-dashboard`
|
||||||
|
|
||||||
|
Пример нового чарта:
|
||||||
|
```python
|
||||||
|
{
|
||||||
|
"slice_name": "📊 My New Chart",
|
||||||
|
"viz_type": "echarts_bar",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"x_axis": "event_type",
|
||||||
|
"metrics": [{"sqlExpression": "COUNT(*)", "label": "Count"}],
|
||||||
|
"time_range": "No filter"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Superset не стартует
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Проверить логи
|
||||||
|
make logs service=superset
|
||||||
|
|
||||||
|
# Перезапуск
|
||||||
|
make superset-restart
|
||||||
|
|
||||||
|
# Полная переинициализация
|
||||||
|
docker compose down -v
|
||||||
|
docker compose up -d
|
||||||
|
make superset-init
|
||||||
|
```
|
||||||
|
|
||||||
|
### Нет данных в чартах
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Проверить данные в ClickHouse
|
||||||
|
docker compose exec clickhouse clickhouse-client -q "SELECT count() FROM dm.v_events_enriched"
|
||||||
|
|
||||||
|
# Перезапустить ETL
|
||||||
|
make transform
|
||||||
|
```
|
||||||
|
|
||||||
|
### Ошибка подключения к ClickHouse
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Проверить доступность ClickHouse
|
||||||
|
docker compose exec superset bash -c "ping clickhouse"
|
||||||
|
|
||||||
|
# Проверить порт
|
||||||
|
docker compose exec superset bash -c "curl clickhouse:8123"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Порты сервисов
|
||||||
|
|
||||||
|
| Сервис | URL | Логин/Пароль |
|
||||||
|
|--------|-----|--------------|
|
||||||
|
| Superset | http://localhost:8088 | admin / admin |
|
||||||
|
| ClickHouse HTTP | http://localhost:9123 | default / (пустой) |
|
||||||
|
| Airflow | http://localhost:8080 | admin / admin |
|
||||||
|
| Grafana | http://localhost:3000 | admin / admin |
|
||||||
|
| Prometheus | http://localhost:9090 | - |
|
||||||
|
| Kafka UI | http://localhost:8082 | - |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Дополнительные ресурсы
|
||||||
|
|
||||||
|
- [Superset Documentation](https://superset.apache.org/docs/intro)
|
||||||
|
- [ClickHouse SQL Reference](https://clickhouse.com/docs/en/sql-reference)
|
||||||
|
- [ARCHITECTURE.md](./ARCHITECTURE.md) — архитектура хранилища
|
||||||
@@ -0,0 +1,471 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
================================================================================
|
||||||
|
Скрипт создания дашборда "E-commerce Analytics" в Superset
|
||||||
|
================================================================================
|
||||||
|
Назначение:
|
||||||
|
- Создание чартов (Charts) на основе датасетов DM-слоя
|
||||||
|
- Создание дашборда с布局 и фильтрами
|
||||||
|
- Настройка native filters
|
||||||
|
|
||||||
|
Запуск:
|
||||||
|
Внутри контейнера superset:
|
||||||
|
python /app/superset_init/create_dashboard.py
|
||||||
|
|
||||||
|
Чарты которые создаются:
|
||||||
|
1. KPI блок (4 Big Number): Total Events, Unique Users, Sessions, Avg/Sess
|
||||||
|
2. Динамика: Events by Hour (Line), Traffic by Device (Pie)
|
||||||
|
3. География: World Map по странам
|
||||||
|
4. Маркетинг: UTM Source/Medium Table, Top Pages Bar
|
||||||
|
5. Качество данных: DQ Summary Bar
|
||||||
|
================================================================================
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
sys.path.insert(0, '/app')
|
||||||
|
|
||||||
|
try:
|
||||||
|
from superset.app import create_app
|
||||||
|
from superset.extensions import db, security_manager
|
||||||
|
from superset.connectors.sqla.models import SqlaTable
|
||||||
|
from superset.charts.data_access_layer import ChartDAO
|
||||||
|
from superset.dashboards.data_access_layer import DashboardDAO
|
||||||
|
from superset.charts.schemas import ChartPostSchema
|
||||||
|
from superset.dashboards.schemas import DashboardPostSchema
|
||||||
|
from superset.commands.chart.create import CreateChartCommand
|
||||||
|
from superset.commands.dashboard.create import CreateDashboardCommand
|
||||||
|
from superset.utils.core import DatasourceType
|
||||||
|
except ImportError as e:
|
||||||
|
logger.error(f"Failed to import Superset modules: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
# Конфигурация чартов
|
||||||
|
CHARTS_CONFIG = [
|
||||||
|
# KPI блок
|
||||||
|
{
|
||||||
|
"slice_name": "📊 Total Events",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"metric": {
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(*)",
|
||||||
|
"column": None,
|
||||||
|
"aggregate": None,
|
||||||
|
"label": "Total Events",
|
||||||
|
"optionName": "metric_1"
|
||||||
|
},
|
||||||
|
"y_axis_format": ",d",
|
||||||
|
"show_trend_line": False,
|
||||||
|
"time_range": "No filter"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slice_name": "👤 Unique Users",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"metric": {
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(DISTINCT user_domain_id)",
|
||||||
|
"label": "Unique Users",
|
||||||
|
"optionName": "metric_2"
|
||||||
|
},
|
||||||
|
"y_axis_format": ",d",
|
||||||
|
"show_trend_line": False,
|
||||||
|
"time_range": "No filter"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slice_name": "🎯 Unique Sessions",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"metric": {
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(DISTINCT click_id)",
|
||||||
|
"label": "Unique Sessions",
|
||||||
|
"optionName": "metric_3"
|
||||||
|
},
|
||||||
|
"y_axis_format": ",d",
|
||||||
|
"show_trend_line": False,
|
||||||
|
"time_range": "No filter"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slice_name": "📈 Avg Events/Session",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"metric": {
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(*) / COUNT(DISTINCT click_id)",
|
||||||
|
"label": "Avg Events/Session",
|
||||||
|
"optionName": "metric_4"
|
||||||
|
},
|
||||||
|
"y_axis_format": ".2f",
|
||||||
|
"show_trend_line": False,
|
||||||
|
"time_range": "No filter"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
# Динамика
|
||||||
|
{
|
||||||
|
"slice_name": "📅 Events by Hour",
|
||||||
|
"viz_type": "echarts_timeseries_line",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"granularity_sqla": "event_ts",
|
||||||
|
"time_grain_sqla": "PT1H",
|
||||||
|
"metrics": [
|
||||||
|
{
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(*)",
|
||||||
|
"label": "Events"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"groupby": [],
|
||||||
|
"time_range": "Last week",
|
||||||
|
"adhoc_filters": [],
|
||||||
|
"row_limit": 10000
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slice_name": "📱 Traffic by Device",
|
||||||
|
"viz_type": "pie",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"groupby": ["device_type"],
|
||||||
|
"metric": {
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(*)",
|
||||||
|
"label": "Count"
|
||||||
|
},
|
||||||
|
"row_limit": 100,
|
||||||
|
"donut": True,
|
||||||
|
"show_legend": True,
|
||||||
|
"labels_outside": True,
|
||||||
|
"time_range": "No filter"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
# География
|
||||||
|
{
|
||||||
|
"slice_name": "🌍 Geography Map",
|
||||||
|
"viz_type": "world_map",
|
||||||
|
"dataset_name": "v_events_enriched",
|
||||||
|
"params": {
|
||||||
|
"entity": "geo_country",
|
||||||
|
"metric": {
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(*)",
|
||||||
|
"label": "Events"
|
||||||
|
},
|
||||||
|
"row_limit": 500,
|
||||||
|
"linear_color_scheme": "blue_white_yellow",
|
||||||
|
"time_range": "No filter"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
# Маркетинг
|
||||||
|
{
|
||||||
|
"slice_name": "🔗 UTM Effectiveness Table",
|
||||||
|
"viz_type": "table",
|
||||||
|
"dataset_name": "v_utm_effectiveness",
|
||||||
|
"params": {
|
||||||
|
"groupby": ["utm_source", "utm_medium", "utm_campaign"],
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(clicks)", "label": "Clicks"},
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(uniq_users)", "label": "Users"},
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(uniq_sessions)", "label": "Sessions"}
|
||||||
|
],
|
||||||
|
"row_limit": 100,
|
||||||
|
"time_range": "No filter",
|
||||||
|
"adhoc_filters": [
|
||||||
|
{
|
||||||
|
"clause": "WHERE",
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "utm_source IS NOT NULL",
|
||||||
|
"subject": None,
|
||||||
|
"operator": None,
|
||||||
|
"comparator": None
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"slice_name": "📄 Top Pages",
|
||||||
|
"viz_type": "echarts_bar",
|
||||||
|
"dataset_name": "v_top_pages_daily",
|
||||||
|
"params": {
|
||||||
|
"x_axis": "page_url_path",
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(pageviews)", "label": "Pageviews"}
|
||||||
|
],
|
||||||
|
"row_limit": 20,
|
||||||
|
"order_by_cols": [["SUM(pageviews)", False]],
|
||||||
|
"time_range": "No filter",
|
||||||
|
"orientation": "vertical",
|
||||||
|
"show_legend": False
|
||||||
|
}
|
||||||
|
},
|
||||||
|
# Качество данных
|
||||||
|
{
|
||||||
|
"slice_name": "🔍 Data Quality Summary",
|
||||||
|
"viz_type": "echarts_bar",
|
||||||
|
"dataset_name": "dq_summary",
|
||||||
|
"params": {
|
||||||
|
"x_axis": "layer",
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(check_value)", "label": "Row Count"}
|
||||||
|
],
|
||||||
|
"adhoc_filters": [
|
||||||
|
{
|
||||||
|
"clause": "WHERE",
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "check_name = 'total_rows'",
|
||||||
|
"subject": None,
|
||||||
|
"operator": None,
|
||||||
|
"comparator": None
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"row_limit": 100,
|
||||||
|
"time_range": "No filter",
|
||||||
|
"show_legend": False
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
# Конфигурация дашборда
|
||||||
|
DASHBOARD_CONFIG = {
|
||||||
|
"dashboard_title": "🛒 E-commerce Analytics Dashboard",
|
||||||
|
"description": "Аналитический дашборд для e-commerce кликстрима. Показывает трафик, конверсии, географию и качество данных.",
|
||||||
|
"published": True,
|
||||||
|
"slug": "ecommerce-analytics",
|
||||||
|
"json_metadata": json.dumps({
|
||||||
|
"native_filter_configuration": [
|
||||||
|
{
|
||||||
|
"id": "date_filter",
|
||||||
|
"name": "📅 Date Range",
|
||||||
|
"filterType": "filter_time",
|
||||||
|
"targets": [{"datasetId": None, "column": {"name": "event_date"}}],
|
||||||
|
"defaultValue": "Last week",
|
||||||
|
"scope": {"root": ["ROOT_ID"], "excluded": []},
|
||||||
|
"cascadeParentIds": [],
|
||||||
|
"isInstant": True
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "country_filter",
|
||||||
|
"name": "🌍 Country",
|
||||||
|
"filterType": "filter_select",
|
||||||
|
"targets": [{"datasetId": None, "column": {"name": "geo_country"}}],
|
||||||
|
"scope": {"root": ["ROOT_ID"], "excluded": []},
|
||||||
|
"isInstant": True,
|
||||||
|
"allowsMultipleValues": True,
|
||||||
|
"isRequired": False
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "device_filter",
|
||||||
|
"name": "📱 Device Type",
|
||||||
|
"filterType": "filter_select",
|
||||||
|
"targets": [{"datasetId": None, "column": {"name": "device_type"}}],
|
||||||
|
"scope": {"root": ["ROOT_ID"], "excluded": []},
|
||||||
|
"isInstant": True,
|
||||||
|
"allowsMultipleValues": True,
|
||||||
|
"isRequired": False
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "browser_filter",
|
||||||
|
"name": "🌐 Browser",
|
||||||
|
"filterType": "filter_select",
|
||||||
|
"targets": [{"datasetId": None, "column": {"name": "browser_name"}}],
|
||||||
|
"scope": {"root": ["ROOT_ID"], "excluded": []},
|
||||||
|
"isInstant": True,
|
||||||
|
"allowsMultipleValues": True,
|
||||||
|
"isRequired": False
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"color_scheme": "supersetColors",
|
||||||
|
"label_colors": {}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_dataset_by_name(app, dataset_name: str) -> SqlaTable:
|
||||||
|
"""Получение датасета по имени таблицы"""
|
||||||
|
with app.app_context():
|
||||||
|
dataset = db.session.query(SqlaTable).filter_by(
|
||||||
|
table_name=dataset_name,
|
||||||
|
schema="dm"
|
||||||
|
).first()
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
|
def create_chart(app, chart_config: dict, dataset: SqlaTable) -> Optional[dict]:
|
||||||
|
"""Создание чарта"""
|
||||||
|
with app.app_context():
|
||||||
|
try:
|
||||||
|
# Проверяем, существует ли чарт
|
||||||
|
from superset.charts.data_access_layer import ChartDAO
|
||||||
|
existing = ChartDAO.find_by_title(chart_config["slice_name"])
|
||||||
|
|
||||||
|
if existing:
|
||||||
|
logger.info(f"Chart '{chart_config['slice_name']}' already exists")
|
||||||
|
return {"id": existing.id, "title": existing.slice_name}
|
||||||
|
|
||||||
|
# Подготавливаем параметры
|
||||||
|
params = chart_config["params"].copy()
|
||||||
|
params["datasource"] = f"{dataset.id}__{DatasourceType.TABLE.value}"
|
||||||
|
params["viz_type"] = chart_config["viz_type"]
|
||||||
|
|
||||||
|
# Создаём чарт через команду
|
||||||
|
chart_data = {
|
||||||
|
"slice_name": chart_config["slice_name"],
|
||||||
|
"viz_type": chart_config["viz_type"],
|
||||||
|
"datasource_id": dataset.id,
|
||||||
|
"datasource_type": DatasourceType.TABLE.value,
|
||||||
|
"params": json.dumps(params),
|
||||||
|
"description": f"Chart created automatically for {chart_config['dataset_name']}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Используем прямой SQL для создания
|
||||||
|
from superset.charts.commands.create import CreateChartCommand
|
||||||
|
|
||||||
|
result = CreateChartCommand(chart_data).run()
|
||||||
|
logger.info(f"Created chart: {chart_config['slice_name']} (ID: {result.id})")
|
||||||
|
return {"id": result.id, "title": result.slice_name}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to create chart '{chart_config['slice_name']}': {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def create_dashboard(app, charts: list):
|
||||||
|
"""Создание дашборда с чартами"""
|
||||||
|
with app.app_context():
|
||||||
|
try:
|
||||||
|
# Проверяем, существует ли дашборд
|
||||||
|
from superset.dashboards.data_access_layer import DashboardDAO
|
||||||
|
existing = DashboardDAO.get_by_slug(DASHBOARD_CONFIG["slug"])
|
||||||
|
|
||||||
|
if existing:
|
||||||
|
logger.info(f"Dashboard '{DASHBOARD_CONFIG['dashboard_title']}' already exists")
|
||||||
|
return existing
|
||||||
|
|
||||||
|
# Создаём позиции чартов для layout
|
||||||
|
positions = {
|
||||||
|
"DASHBOARD_VERSION_KEY": "v2"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Добавляем чарты в layout (grid: 12 columns)
|
||||||
|
# Row 1: KPI блок (4 чарта по 3 колонки)
|
||||||
|
# Row 2: Events by Hour (8) | Geography (4)
|
||||||
|
# Row 3: Traffic by Device (4) | Top Pages (8)
|
||||||
|
# Row 4: UTM Table (12)
|
||||||
|
# Row 5: DQ Summary (12)
|
||||||
|
|
||||||
|
y_position = 0
|
||||||
|
chart_index = 0
|
||||||
|
|
||||||
|
for chart in charts:
|
||||||
|
if chart:
|
||||||
|
positions[f"CHART-{chart['id']}"] = {
|
||||||
|
"id": f"CHART-{chart['id']}",
|
||||||
|
"type": "CHART",
|
||||||
|
"parents": ["ROOT_ID"],
|
||||||
|
"meta": {
|
||||||
|
"chartId": chart['id'],
|
||||||
|
"sliceName": chart['title'],
|
||||||
|
"height": 50,
|
||||||
|
"width": 4 if chart_index < 4 else 6, # KPI - по 4, остальные - по 6
|
||||||
|
"x": (chart_index % 3) * 4 if chart_index < 4 else (chart_index % 2) * 6,
|
||||||
|
"y": y_position
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chart_index += 1
|
||||||
|
if chart_index % 4 == 0:
|
||||||
|
y_position += 50
|
||||||
|
|
||||||
|
# Создаём дашборд
|
||||||
|
dashboard_data = {
|
||||||
|
"dashboard_title": DASHBOARD_CONFIG["dashboard_title"],
|
||||||
|
"slug": DASHBOARD_CONFIG["slug"],
|
||||||
|
"description": DASHBOARD_CONFIG["description"],
|
||||||
|
"published": DASHBOARD_CONFIG["published"],
|
||||||
|
"json_metadata": DASHBOARD_CONFIG["json_metadata"],
|
||||||
|
"position_json": json.dumps(positions)
|
||||||
|
}
|
||||||
|
|
||||||
|
from superset.dashboards.commands.create import CreateDashboardCommand
|
||||||
|
result = CreateDashboardCommand(dashboard_data).run()
|
||||||
|
|
||||||
|
# Добавляем чарты к дашборду
|
||||||
|
from superset.dashboards.dao import DashboardDAO
|
||||||
|
dashboard = DashboardDAO.get_by_id(result.id)
|
||||||
|
|
||||||
|
from superset.charts.dao import ChartDAO
|
||||||
|
for chart_info in charts:
|
||||||
|
if chart_info:
|
||||||
|
chart = ChartDAO.find_by_id(chart_info["id"])
|
||||||
|
if chart:
|
||||||
|
dashboard.slices.append(chart)
|
||||||
|
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
logger.info(f"Created dashboard: {DASHBOARD_CONFIG['dashboard_title']} (ID: {result.id})")
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to create dashboard: {e}")
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Главная функция"""
|
||||||
|
logger.info("=" * 60)
|
||||||
|
logger.info("Creating E-commerce Analytics Dashboard")
|
||||||
|
logger.info("=" * 60)
|
||||||
|
|
||||||
|
app = create_app()
|
||||||
|
|
||||||
|
created_charts = []
|
||||||
|
|
||||||
|
# Создаём чарты
|
||||||
|
for chart_config in CHARTS_CONFIG:
|
||||||
|
dataset = get_dataset_by_name(app, chart_config["dataset_name"])
|
||||||
|
if not dataset:
|
||||||
|
logger.warning(f"Dataset '{chart_config['dataset_name']}' not found, skipping chart")
|
||||||
|
continue
|
||||||
|
|
||||||
|
chart = create_chart(app, chart_config, dataset)
|
||||||
|
if chart:
|
||||||
|
created_charts.append(chart)
|
||||||
|
|
||||||
|
logger.info(f"Created {len(created_charts)} charts")
|
||||||
|
|
||||||
|
# Создаём дашборд
|
||||||
|
if created_charts:
|
||||||
|
dashboard = create_dashboard(app, created_charts)
|
||||||
|
if dashboard:
|
||||||
|
logger.info("=" * 60)
|
||||||
|
logger.info("Dashboard created successfully!")
|
||||||
|
logger.info(f"Dashboard URL: /superset/dashboard/{dashboard.id}/")
|
||||||
|
logger.info("=" * 60)
|
||||||
|
else:
|
||||||
|
logger.error("Failed to create dashboard")
|
||||||
|
else:
|
||||||
|
logger.error("No charts created, cannot create dashboard")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
{
|
||||||
|
"dashboards": [
|
||||||
|
{
|
||||||
|
"__Dashboard__": {
|
||||||
|
"dashboard_title": "🛒 E-commerce Analytics Dashboard",
|
||||||
|
"description": "Аналитический дашборд для e-commerce кликстрима: трафик, конверсии, география и качество данных.",
|
||||||
|
"slug": "ecommerce-analytics",
|
||||||
|
"published": true,
|
||||||
|
"json_metadata": "{\"native_filter_configuration\": [{\"id\": \"date_filter\", \"name\": \"📅 Date Range\", \"filterType\": \"filter_time\", \"targets\": [{\"datasetId\": null, \"column\": {\"name\": \"event_date\"}}], \"defaultValue\": \"Last week\", \"scope\": {\"root\": [\"ROOT_ID\"], \"excluded\": []}, \"cascadeParentIds\": [], \"isInstant\": true}, {\"id\": \"country_filter\", \"name\": \"🌍 Country\", \"filterType\": \"filter_select\", \"targets\": [{\"datasetId\": null, \"column\": {\"name\": \"geo_country\"}}], \"scope\": {\"root\": [\"ROOT_ID\"], \"excluded\": []}, \"isInstant\": true, \"allowsMultipleValues\": true, \"isRequired\": false}, {\"id\": \"device_filter\", \"name\": \"📱 Device Type\", \"filterType\": \"filter_select\", \"targets\": [{\"datasetId\": null, \"column\": {\"name\": \"device_type\"}}], \"scope\": {\"root\": [\"ROOT_ID\"], \"excluded\": []}, \"isInstant\": true, \"allowsMultipleValues\": true, \"isRequired\": false}, {\"id\": \"browser_filter\", \"name\": \"🌐 Browser\", \"filterType\": \"filter_select\", \"targets\": [{\"datasetId\": null, \"column\": {\"name\": \"browser_name\"}}], \"scope\": {\"root\": [\"ROOT_ID\"], \"excluded\": []}, \"isInstant\": true, \"allowsMultipleValues\": true, \"isRequired\": false}], \"color_scheme\": \"supersetColors\", \"label_colors\": {}}",
|
||||||
|
"position_json": "{\"DASHBOARD_VERSION_KEY\": \"v2\", \"CHART-1\": {\"id\": \"CHART-1\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 1, \"sliceName\": \"📊 Total Events\", \"height\": 50, \"width\": 4, \"x\": 0, \"y\": 0}}, \"CHART-2\": {\"id\": \"CHART-2\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 2, \"sliceName\": \"👤 Unique Users\", \"height\": 50, \"width\": 4, \"x\": 4, \"y\": 0}}, \"CHART-3\": {\"id\": \"CHART-3\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 3, \"sliceName\": \"🎯 Unique Sessions\", \"height\": 50, \"width\": 4, \"x\": 8, \"y\": 0}}, \"CHART-4\": {\"id\": \"CHART-4\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 4, \"sliceName\": \"📈 Avg Events/Session\", \"height\": 50, \"width\": 4, \"x\": 0, \"y\": 50}}, \"CHART-5\": {\"id\": \"CHART-5\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 5, \"sliceName\": \"📅 Events by Hour\", \"height\": 50, \"width\": 8, \"x\": 0, \"y\": 100}}, \"CHART-6\": {\"id\": \"CHART-6\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 6, \"sliceName\": \"📱 Traffic by Device\", \"height\": 50, \"width\": 4, \"x\": 8, \"y\": 100}}, \"CHART-7\": {\"id\": \"CHART-7\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 7, \"sliceName\": \"🌍 Geography Map\", \"height\": 50, \"width\": 6, \"x\": 0, \"y\": 150}}, \"CHART-8\": {\"id\": \"CHART-8\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 8, \"sliceName\": \"🔗 UTM Effectiveness Table\", \"height\": 50, \"width\": 6, \"x\": 6, \"y\": 150}}, \"CHART-9\": {\"id\": \"CHART-9\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 9, \"sliceName\": \"📄 Top Pages\", \"height\": 50, \"width\": 6, \"x\": 0, \"y\": 200}}, \"CHART-10\": {\"id\": \"CHART-10\", \"type\": \"CHART\", \"parents\": [\"ROOT_ID\"], \"meta\": {\"chartId\": 10, \"sliceName\": \"🔍 Data Quality Summary\", \"height\": 50, \"width\": 6, \"x\": 6, \"y\": 200}}}",
|
||||||
|
"slices": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"charts": [
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "📊 Total Events",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_events_enriched",
|
||||||
|
"params": "{\"datasource\": \"1__table\", \"viz_type\": \"big_number\", \"metric\": {\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(*)\", \"column\": null, \"aggregate\": null, \"label\": \"Total Events\", \"optionName\": \"metric_1\"}, \"y_axis_format\": \",d\", \"show_trend_line\": false, \"time_range\": \"No filter\"}",
|
||||||
|
"description": "Общее количество событий"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "👤 Unique Users",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_events_enriched",
|
||||||
|
"params": "{\"datasource\": \"1__table\", \"viz_type\": \"big_number\", \"metric\": {\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(DISTINCT user_domain_id)\", \"label\": \"Unique Users\", \"optionName\": \"metric_2\"}, \"y_axis_format\": \",d\", \"show_trend_line\": false, \"time_range\": \"No filter\"}",
|
||||||
|
"description": "Уникальные пользователи"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "🎯 Unique Sessions",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_events_enriched",
|
||||||
|
"params": "{\"datasource\": \"1__table\", \"viz_type\": \"big_number\", \"metric\": {\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(DISTINCT click_id)\", \"label\": \"Unique Sessions\", \"optionName\": \"metric_3\"}, \"y_axis_format\": \",d\", \"show_trend_line\": false, \"time_range\": \"No filter\"}",
|
||||||
|
"description": "Уникальные сессии"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "📈 Avg Events/Session",
|
||||||
|
"viz_type": "big_number",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_events_enriched",
|
||||||
|
"params": "{\"datasource\": \"1__table\", \"viz_type\": \"big_number\", \"metric\": {\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(*) / COUNT(DISTINCT click_id)\", \"label\": \"Avg Events/Session\", \"optionName\": \"metric_4\"}, \"y_axis_format\": \".2f\", \"show_trend_line\": false, \"time_range\": \"No filter\"}",
|
||||||
|
"description": "Среднее количество событий на сессию"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "📅 Events by Hour",
|
||||||
|
"viz_type": "echarts_timeseries_line",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_events_enriched",
|
||||||
|
"params": "{\"datasource\": \"1__table\", \"viz_type\": \"echarts_timeseries_line\", \"granularity_sqla\": \"event_ts\", \"time_grain_sqla\": \"PT1H\", \"metrics\": [{\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(*)\", \"label\": \"Events\"}], \"groupby\": [], \"time_range\": \"Last week\", \"adhoc_filters\": [], \"row_limit\": 10000}",
|
||||||
|
"description": "События по часам"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "📱 Traffic by Device",
|
||||||
|
"viz_type": "pie",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_events_enriched",
|
||||||
|
"params": "{\"datasource\": \"1__table\", \"viz_type\": \"pie\", \"groupby\": [\"device_type\"], \"metric\": {\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(*)\", \"label\": \"Count\"}, \"row_limit\": 100, \"donut\": true, \"show_legend\": true, \"labels_outside\": true, \"time_range\": \"No filter\"}",
|
||||||
|
"description": "Распределение трафика по устройствам"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "🌍 Geography Map",
|
||||||
|
"viz_type": "world_map",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_events_enriched",
|
||||||
|
"params": "{\"datasource\": \"1__table\", \"viz_type\": \"world_map\", \"entity\": \"geo_country\", \"metric\": {\"expressionType\": \"SQL\", \"sqlExpression\": \"COUNT(*)\", \"label\": \"Events\"}, \"row_limit\": 500, \"linear_color_scheme\": \"blue_white_yellow\", \"time_range\": \"No filter\"}",
|
||||||
|
"description": "География посетителей"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "🔗 UTM Effectiveness Table",
|
||||||
|
"viz_type": "table",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_utm_effectiveness",
|
||||||
|
"params": "{\"datasource\": \"2__table\", \"viz_type\": \"table\", \"groupby\": [\"utm_source\", \"utm_medium\", \"utm_campaign\"], \"metrics\": [{\"expressionType\": \"SQL\", \"sqlExpression\": \"SUM(clicks)\", \"label\": \"Clicks\"}, {\"expressionType\": \"SQL\", \"sqlExpression\": \"SUM(uniq_users)\", \"label\": \"Users\"}, {\"expressionType\": \"SQL\", \"sqlExpression\": \"SUM(uniq_sessions)\", \"label\": \"Sessions\"}], \"row_limit\": 100, \"time_range\": \"No filter\", \"adhoc_filters\": [{\"clause\": \"WHERE\", \"expressionType\": \"SQL\", \"sqlExpression\": \"utm_source IS NOT NULL\", \"subject\": null, \"operator\": null, \"comparator\": null}]}\n",
|
||||||
|
"description": "Эффективность UTM-кампаний"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "📄 Top Pages",
|
||||||
|
"viz_type": "echarts_bar",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.v_top_pages_daily",
|
||||||
|
"params": "{\"datasource\": \"3__table\", \"viz_type\": \"echarts_bar\", \"x_axis\": \"page_url_path\", \"metrics\": [{\"expressionType\": \"SQL\", \"sqlExpression\": \"SUM(pageviews)\", \"label\": \"Pageviews\"}], \"row_limit\": 20, \"order_by_cols\": [[\"SUM(pageviews)\", false]], \"time_range\": \"No filter\", \"orientation\": \"vertical\", \"show_legend\": false}",
|
||||||
|
"description": "Топ страниц по просмотрам"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": "🔍 Data Quality Summary",
|
||||||
|
"viz_type": "echarts_bar",
|
||||||
|
"datasource_type": "table",
|
||||||
|
"datasource_name": "dm.dq_summary",
|
||||||
|
"params": "{\"datasource\": \"4__table\", \"viz_type\": \"echarts_bar\", \"x_axis\": \"layer\", \"metrics\": [{\"expressionType\": \"SQL\", \"sqlExpression\": \"SUM(check_value)\", \"label\": \"Row Count\"}], \"adhoc_filters\": [{\"clause\": \"WHERE\", \"expressionType\": \"SQL\", \"sqlExpression\": \"check_name = 'total_rows'\", \"subject\": null, \"operator\": null, \"comparator\": null}], \"row_limit\": 100, \"time_range\": \"No filter\", \"show_legend\": false}",
|
||||||
|
"description": "Сводка по качеству данных"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"datasets": [
|
||||||
|
{
|
||||||
|
"__SqlaTable__": {
|
||||||
|
"table_name": "v_events_enriched",
|
||||||
|
"schema": "dm",
|
||||||
|
"database": "clickhouse_dwh",
|
||||||
|
"description": "Полная обогащённая витрина событий (event + click)",
|
||||||
|
"columns": [
|
||||||
|
{"column_name": "event_id", "type": "UUID", "description": "UUID события"},
|
||||||
|
{"column_name": "event_ts", "type": "DateTime64(6)", "description": "Время события"},
|
||||||
|
{"column_name": "event_date", "type": "Date", "description": "Дата события"},
|
||||||
|
{"column_name": "event_type", "type": "String", "description": "Тип события"},
|
||||||
|
{"column_name": "click_id", "type": "UUID", "description": "ID сессии"},
|
||||||
|
{"column_name": "user_domain_id", "type": "UUID", "description": "ID пользователя"},
|
||||||
|
{"column_name": "device_type", "type": "String", "description": "Тип устройства"},
|
||||||
|
{"column_name": "geo_country", "type": "String", "description": "Страна"},
|
||||||
|
{"column_name": "browser_name", "type": "String", "description": "Браузер"},
|
||||||
|
{"column_name": "utm_source", "type": "String", "description": "UTM Source"},
|
||||||
|
{"column_name": "utm_medium", "type": "String", "description": "UTM Medium"},
|
||||||
|
{"column_name": "page_url_path", "type": "String", "description": "Путь URL"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__SqlaTable__": {
|
||||||
|
"table_name": "v_utm_effectiveness",
|
||||||
|
"schema": "dm",
|
||||||
|
"database": "clickhouse_dwh",
|
||||||
|
"description": "Эффективность UTM-кампаний",
|
||||||
|
"columns": [
|
||||||
|
{"column_name": "event_date", "type": "Date", "description": "Дата"},
|
||||||
|
{"column_name": "utm_source", "type": "String", "description": "UTM Source"},
|
||||||
|
{"column_name": "utm_medium", "type": "String", "description": "UTM Medium"},
|
||||||
|
{"column_name": "utm_campaign", "type": "String", "description": "UTM Campaign"},
|
||||||
|
{"column_name": "clicks", "type": "UInt64", "description": "Клики"},
|
||||||
|
{"column_name": "uniq_users", "type": "UInt64", "description": "Уникальные пользователи"},
|
||||||
|
{"column_name": "uniq_sessions", "type": "UInt64", "description": "Уникальные сессии"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__SqlaTable__": {
|
||||||
|
"table_name": "v_top_pages_daily",
|
||||||
|
"schema": "dm",
|
||||||
|
"database": "clickhouse_dwh",
|
||||||
|
"description": "Популярность страниц по дням",
|
||||||
|
"columns": [
|
||||||
|
{"column_name": "event_date", "type": "Date", "description": "Дата"},
|
||||||
|
{"column_name": "page_url_path", "type": "String", "description": "Путь URL"},
|
||||||
|
{"column_name": "pageviews", "type": "UInt64", "description": "Просмотры"},
|
||||||
|
{"column_name": "uniq_clicks", "type": "UInt64", "description": "Уникальные клики"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"__SqlaTable__": {
|
||||||
|
"table_name": "dq_summary",
|
||||||
|
"schema": "dm",
|
||||||
|
"database": "clickhouse_dwh",
|
||||||
|
"description": "Сводка по качеству данных",
|
||||||
|
"columns": [
|
||||||
|
{"column_name": "check_date", "type": "Date", "description": "Дата проверки"},
|
||||||
|
{"column_name": "layer", "type": "String", "description": "Слой (stg/ods/dds)"},
|
||||||
|
{"column_name": "table_name", "type": "String", "description": "Имя таблицы"},
|
||||||
|
{"column_name": "check_name", "type": "String", "description": "Тип проверки"},
|
||||||
|
{"column_name": "check_value", "type": "UInt64", "description": "Значение"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"databases": [
|
||||||
|
{
|
||||||
|
"__Database__": {
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"sqlalchemy_uri": "clickhouse+native://default@clickhouse:9000/default",
|
||||||
|
"expose_in_sqllab": true,
|
||||||
|
"allow_ctas": false,
|
||||||
|
"allow_cvas": false,
|
||||||
|
"allow_dml": false,
|
||||||
|
"allow_file_upload": false,
|
||||||
|
"extra": "{}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
================================================================================
|
||||||
|
Экспорт дашборда Superset в JSON-формат
|
||||||
|
================================================================================
|
||||||
|
Назначение:
|
||||||
|
- Экспорт созданного дашборда в JSON для версионирования
|
||||||
|
- Формат совместимый с superset import-dashboards
|
||||||
|
|
||||||
|
Запуск:
|
||||||
|
docker compose exec superset python /app/superset_init/export_dashboard.py
|
||||||
|
================================================================================
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
sys.path.insert(0, '/app')
|
||||||
|
|
||||||
|
try:
|
||||||
|
from superset.app import create_app
|
||||||
|
from superset.dashboards.data_access_layer import DashboardDAO
|
||||||
|
from superset.charts.data_access_layer import ChartDAO
|
||||||
|
except ImportError as e:
|
||||||
|
print(f"Error importing: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def export_dashboard(slug: str, output_path: str):
|
||||||
|
"""Экспорт дашборда в JSON"""
|
||||||
|
app = create_app()
|
||||||
|
|
||||||
|
with app.app_context():
|
||||||
|
dashboard = DashboardDAO.get_by_slug(slug)
|
||||||
|
|
||||||
|
if not dashboard:
|
||||||
|
print(f"Dashboard with slug '{slug}' not found")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Собираем данные дашборда
|
||||||
|
dashboard_data = {
|
||||||
|
"dashboards": [
|
||||||
|
{
|
||||||
|
"__Dashboard__": {
|
||||||
|
"dashboard_title": dashboard.dashboard_title,
|
||||||
|
"description": dashboard.description,
|
||||||
|
"slug": dashboard.slug,
|
||||||
|
"json_metadata": dashboard.json_metadata,
|
||||||
|
"position_json": dashboard.position_json,
|
||||||
|
"published": dashboard.published,
|
||||||
|
"slices": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"charts": [],
|
||||||
|
"datasets": []
|
||||||
|
}
|
||||||
|
|
||||||
|
# Добавляем чарты
|
||||||
|
for slice_obj in dashboard.slices:
|
||||||
|
chart_data = {
|
||||||
|
"__Slice__": {
|
||||||
|
"slice_name": slice_obj.slice_name,
|
||||||
|
"viz_type": slice_obj.viz_type,
|
||||||
|
"params": slice_obj.params,
|
||||||
|
"description": slice_obj.description,
|
||||||
|
"datasource_type": slice_obj.datasource_type,
|
||||||
|
"datasource_name": slice_obj.datasource.name if slice_obj.datasource else None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dashboard_data["dashboards"][0]["__Dashboard__"]["slices"].append(slice_obj.id)
|
||||||
|
dashboard_data["charts"].append(chart_data)
|
||||||
|
|
||||||
|
# Добавляем датасет
|
||||||
|
if slice_obj.datasource:
|
||||||
|
ds = slice_obj.datasource
|
||||||
|
dataset_data = {
|
||||||
|
"__SqlaTable__": {
|
||||||
|
"table_name": ds.table_name,
|
||||||
|
"schema": ds.schema,
|
||||||
|
"database": ds.database.database_name if ds.database else None,
|
||||||
|
"description": ds.description,
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"column_name": col.column_name,
|
||||||
|
"type": col.type,
|
||||||
|
"description": col.description
|
||||||
|
}
|
||||||
|
for col in ds.columns
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# Добавляем уникальные датасеты
|
||||||
|
if dataset_data not in dashboard_data["datasets"]:
|
||||||
|
dashboard_data["datasets"].append(dataset_data)
|
||||||
|
|
||||||
|
# Сохраняем в файл
|
||||||
|
with open(output_path, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(dashboard_data, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
print(f"Dashboard exported to: {output_path}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
output_file = "/app/superset_init/dashboards/ecommerce_analytics.json"
|
||||||
|
export_dashboard("ecommerce-analytics", output_file)
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
================================================================================
|
||||||
|
Скрипт инициализации Superset для проекта ClickHouse Mini DWH
|
||||||
|
================================================================================
|
||||||
|
Назначение:
|
||||||
|
- Создание подключения к ClickHouse (Database connection)
|
||||||
|
- Импорт датасетов из витрин DM-слоя
|
||||||
|
- Импорт чартов и дашбордов
|
||||||
|
|
||||||
|
Запуск:
|
||||||
|
Внутри контейнера superset:
|
||||||
|
python /app/superset_init/init_superset.py
|
||||||
|
|
||||||
|
Требования:
|
||||||
|
- Запущенный ClickHouse с созданными витринами в схеме dm
|
||||||
|
- Superset инициализирован (superset db upgrade, admin создан)
|
||||||
|
================================================================================
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
# Настройка логирования
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||||
|
)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Добавляем путь к superset
|
||||||
|
sys.path.insert(0, '/app')
|
||||||
|
|
||||||
|
try:
|
||||||
|
from superset.app import create_app
|
||||||
|
from superset.extensions import db
|
||||||
|
from superset.models.core import Database
|
||||||
|
from superset.connectors.sqla.models import SqlaTable, TableColumn
|
||||||
|
from superset.charts.data_access_layer import ChartDAO
|
||||||
|
from superset.dashboards.data_access_layer import DashboardDAO
|
||||||
|
from superset.commands.dataset.create import CreateDatasetCommand
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
except ImportError as e:
|
||||||
|
logger.error(f"Failed to import Superset modules: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Конфигурация подключения к ClickHouse
|
||||||
|
CLICKHOUSE_CONFIG = {
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"sqlalchemy_uri": "clickhouse+native://default@clickhouse:9000/default",
|
||||||
|
"expose_in_sqllab": True,
|
||||||
|
"allow_ctas": False,
|
||||||
|
"allow_cvas": False,
|
||||||
|
"allow_dml": False,
|
||||||
|
"allow_file_upload": False,
|
||||||
|
"extra": json.dumps({
|
||||||
|
"engine_params": {},
|
||||||
|
"metadata_params": {},
|
||||||
|
"schemas_allowed_for_file_upload": []
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
# Датасеты для импорта из DM-слоя
|
||||||
|
DATASETS = [
|
||||||
|
{
|
||||||
|
"table_name": "v_events_enriched",
|
||||||
|
"schema": "dm",
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"description": "Полная обогащённая витрина событий (event + click)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"table_name": "v_daily_traffic",
|
||||||
|
"schema": "dm",
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"description": "Агрегация трафика по дням и измерениям"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"table_name": "v_utm_effectiveness",
|
||||||
|
"schema": "dm",
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"description": "Эффективность UTM-кампаний"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"table_name": "v_top_pages_daily",
|
||||||
|
"schema": "dm",
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"description": "Популярность страниц по дням"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"table_name": "v_session_overview",
|
||||||
|
"schema": "dm",
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"description": "Обзор сессий пользователей"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"table_name": "dq_summary",
|
||||||
|
"schema": "dm",
|
||||||
|
"database_name": "clickhouse_dwh",
|
||||||
|
"description": "Сводка по качеству данных"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def create_clickhouse_connection(app) -> Optional[Database]:
|
||||||
|
"""Создание подключения к ClickHouse"""
|
||||||
|
with app.app_context():
|
||||||
|
logger.info("Creating ClickHouse database connection...")
|
||||||
|
|
||||||
|
# Проверяем, существует ли уже подключение
|
||||||
|
existing = db.session.query(Database).filter_by(
|
||||||
|
database_name=CLICKHOUSE_CONFIG["database_name"]
|
||||||
|
).first()
|
||||||
|
|
||||||
|
if existing:
|
||||||
|
logger.info(f"Database connection '{CLICKHOUSE_CONFIG['database_name']}' already exists")
|
||||||
|
return existing
|
||||||
|
|
||||||
|
try:
|
||||||
|
database = Database(**CLICKHOUSE_CONFIG)
|
||||||
|
db.session.add(database)
|
||||||
|
db.session.commit()
|
||||||
|
logger.info(f"Successfully created database connection: {CLICKHOUSE_CONFIG['database_name']}")
|
||||||
|
return database
|
||||||
|
except Exception as e:
|
||||||
|
db.session.rollback()
|
||||||
|
logger.error(f"Failed to create database connection: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def import_datasets(app):
|
||||||
|
"""Импорт датасетов из DM-слоя"""
|
||||||
|
with app.app_context():
|
||||||
|
logger.info("Importing datasets...")
|
||||||
|
|
||||||
|
# Получаем ID базы данных
|
||||||
|
database = db.session.query(Database).filter_by(
|
||||||
|
database_name="clickhouse_dwh"
|
||||||
|
).first()
|
||||||
|
|
||||||
|
if not database:
|
||||||
|
logger.error("ClickHouse database connection not found")
|
||||||
|
return False
|
||||||
|
|
||||||
|
imported_count = 0
|
||||||
|
for dataset_config in DATASETS:
|
||||||
|
try:
|
||||||
|
# Проверяем, существует ли датасет
|
||||||
|
existing = db.session.query(SqlaTable).filter_by(
|
||||||
|
table_name=dataset_config["table_name"],
|
||||||
|
schema=dataset_config["schema"]
|
||||||
|
).first()
|
||||||
|
|
||||||
|
if existing:
|
||||||
|
logger.info(f"Dataset '{dataset_config['table_name']}' already exists")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Создаём датасет
|
||||||
|
dataset = SqlaTable(
|
||||||
|
table_name=dataset_config["table_name"],
|
||||||
|
schema=dataset_config["schema"],
|
||||||
|
database_id=database.id,
|
||||||
|
database=database,
|
||||||
|
description=dataset_config["description"],
|
||||||
|
is_sqllab_view=False
|
||||||
|
)
|
||||||
|
|
||||||
|
db.session.add(dataset)
|
||||||
|
db.session.flush()
|
||||||
|
|
||||||
|
# Fetch columns from database
|
||||||
|
dataset.fetch_metadata()
|
||||||
|
|
||||||
|
db.session.commit()
|
||||||
|
logger.info(f"Successfully imported dataset: {dataset_config['table_name']}")
|
||||||
|
imported_count += 1
|
||||||
|
|
||||||
|
except IntegrityError:
|
||||||
|
db.session.rollback()
|
||||||
|
logger.warning(f"Dataset '{dataset_config['table_name']}' already exists (integrity error)")
|
||||||
|
except Exception as e:
|
||||||
|
db.session.rollback()
|
||||||
|
logger.error(f"Failed to import dataset '{dataset_config['table_name']}': {e}")
|
||||||
|
|
||||||
|
logger.info(f"Imported {imported_count} new datasets")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Главная функция инициализации"""
|
||||||
|
logger.info("=" * 60)
|
||||||
|
logger.info("Superset Initialization for ClickHouse Mini DWH")
|
||||||
|
logger.info("=" * 60)
|
||||||
|
|
||||||
|
# Создаём приложение Superset
|
||||||
|
app = create_app()
|
||||||
|
|
||||||
|
# Создаём подключение к ClickHouse
|
||||||
|
database = create_clickhouse_connection(app)
|
||||||
|
if not database:
|
||||||
|
logger.error("Failed to create ClickHouse connection")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Импортируем датасеты
|
||||||
|
if not import_datasets(app):
|
||||||
|
logger.error("Failed to import datasets")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
logger.info("=" * 60)
|
||||||
|
logger.info("Superset initialization completed successfully!")
|
||||||
|
logger.info("=" * 60)
|
||||||
|
logger.info("Available datasets:")
|
||||||
|
for ds in DATASETS:
|
||||||
|
logger.info(f" - {ds['schema']}.{ds['table_name']}")
|
||||||
|
logger.info("=" * 60)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user