feat(generator): добавлена популяция пользователей
- Зачем: - steady-stream генератору нужны устойчивые пользователи и возвраты между визитами, чтобы поток был похож на живую модель поведения. - Что: - добавлена ограниченная популяция с кулдауном возврата, ротацией новых пользователей и защитой от второго активного визита. - UUID переведены на единый ГПСЧ генератора, а завершение визита считается по запланированному последнему событию. - добавлены регрессионные тесты и обновлена документация генератора. - Проверка: - uv run --with-requirements generator/requirements.txt pytest generator/tests -q
This commit is contained in:
@@ -36,6 +36,12 @@ class Config:
|
||||
population_max: int = field(
|
||||
default_factory=lambda: int(os.getenv("GEN_POPULATION_MAX", "300"))
|
||||
)
|
||||
p_new_user: float = field(
|
||||
default_factory=lambda: float(os.getenv("GEN_P_NEW_USER", "0.15"))
|
||||
)
|
||||
min_return_minutes: int = field(
|
||||
default_factory=lambda: int(os.getenv("GEN_MIN_RETURN_MINUTES", "30"))
|
||||
)
|
||||
data_dir: Path = field(
|
||||
default_factory=lambda: Path(os.getenv("GEN_DATA_DIR", "/data"))
|
||||
)
|
||||
@@ -68,6 +74,10 @@ class Config:
|
||||
raise ValueError("GEN_MAX_ACTIVE_SESSIONS must be >= 1")
|
||||
if self.population_max < 1:
|
||||
raise ValueError("GEN_POPULATION_MAX must be >= 1")
|
||||
if not 0 <= self.p_new_user <= 1:
|
||||
raise ValueError("GEN_P_NEW_USER must be between 0 and 1")
|
||||
if self.min_return_minutes < 0:
|
||||
raise ValueError("GEN_MIN_RETURN_MINUTES must be >= 0")
|
||||
if self.max_active_sessions >= self.population_max:
|
||||
raise ValueError("GEN_MAX_ACTIVE_SESSIONS must be < GEN_POPULATION_MAX")
|
||||
if not self.data_dir.exists():
|
||||
|
||||
@@ -77,7 +77,7 @@ class EventGenerator:
|
||||
|
||||
def _new_uuid(self) -> str:
|
||||
"""Генерирует новый UUID."""
|
||||
return str(uuid.uuid4())
|
||||
return str(uuid.UUID(int=self.rng.getrandbits(128), version=4))
|
||||
|
||||
def _current_timestamp(self) -> str:
|
||||
"""Возвращает текущую метку времени в формате JSONL."""
|
||||
@@ -133,6 +133,7 @@ class EventGenerator:
|
||||
self,
|
||||
batch_size: int,
|
||||
planned_start_at: datetime | None = None,
|
||||
user_profile: dict[str, dict] | None = None,
|
||||
) -> dict[str, list[dict]]:
|
||||
"""Генерирует один визит с сохранением связей."""
|
||||
if not self.dictionary.browser_events:
|
||||
@@ -177,8 +178,16 @@ class EventGenerator:
|
||||
base_click_id = base_browser["click_id"]
|
||||
base_browser_events = [base_browser for _ in range(len(visit_path))]
|
||||
|
||||
base_device = self.dictionary.device_by_click_id.get(base_click_id)
|
||||
base_geo = self.dictionary.geo_by_click_id.get(base_click_id)
|
||||
base_device = (
|
||||
user_profile["device"]
|
||||
if user_profile is not None
|
||||
else self.dictionary.device_by_click_id.get(base_click_id)
|
||||
)
|
||||
base_geo = (
|
||||
user_profile["geo"]
|
||||
if user_profile is not None
|
||||
else self.dictionary.geo_by_click_id.get(base_click_id)
|
||||
)
|
||||
new_click_id = self._new_uuid()
|
||||
planned_timestamp = planned_start_at or datetime.now(timezone.utc)
|
||||
if planned_timestamp.tzinfo is not None:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Тиковый слой генератора с активными визитами между вызовами."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from clickstream_generator.generation import EventGenerator
|
||||
@@ -10,12 +10,29 @@ from clickstream_generator.generation import EventGenerator
|
||||
TOPICS = ("browser_events", "location_events", "device_events", "geo_events")
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserProfile:
|
||||
"""Постоянный профиль пользователя между визитами."""
|
||||
|
||||
user_domain_id: str
|
||||
seed_click_id: str
|
||||
device: dict
|
||||
geo: dict
|
||||
active_click_id: str | None = None
|
||||
last_finished_at: datetime | None = None
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
return self.active_click_id is not None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActiveVisit:
|
||||
"""Запланированный визит, который выпускается по тикам."""
|
||||
|
||||
batch: dict[str, list[dict]]
|
||||
timestamps: list[datetime]
|
||||
user: UserProfile | None = None
|
||||
next_index: int = 0
|
||||
|
||||
@property
|
||||
@@ -23,6 +40,80 @@ class ActiveVisit:
|
||||
return self.next_index >= len(self.timestamps)
|
||||
|
||||
|
||||
class UserPopulation:
|
||||
"""Ограниченная популяция пользователей для тикового потока."""
|
||||
|
||||
def __init__(self, generator: EventGenerator):
|
||||
self.generator = generator
|
||||
self.users: list[UserProfile] = [
|
||||
self._create_user()
|
||||
for _ in range(generator.config.population_max)
|
||||
]
|
||||
|
||||
def choose_for_visit(self, tick_time: datetime) -> UserProfile | None:
|
||||
"""Выбирает пользователя без активного визита и кулдауна."""
|
||||
available = [
|
||||
user for user in self.users
|
||||
if self._is_available(user, tick_time)
|
||||
]
|
||||
if not available:
|
||||
return self._rotate_new_user()
|
||||
if self.generator.rng.random() < self.generator.config.p_new_user:
|
||||
return self._rotate_new_user() or self.generator.rng.choice(available)
|
||||
return self.generator.rng.choice(available)
|
||||
|
||||
def start_visit(self, user: UserProfile, click_id: str) -> None:
|
||||
user.active_click_id = click_id
|
||||
|
||||
def finish_visit(self, user: UserProfile | None, finished_at: datetime) -> None:
|
||||
if user is None:
|
||||
return
|
||||
user.active_click_id = None
|
||||
user.last_finished_at = finished_at
|
||||
|
||||
def _create_user(self) -> UserProfile:
|
||||
seed_click_id = self.generator.rng.choice(self._profile_seed_click_ids())
|
||||
device = {
|
||||
**self.generator.dictionary.device_by_click_id[seed_click_id],
|
||||
"user_domain_id": self.generator._new_uuid(),
|
||||
}
|
||||
geo = self.generator.dictionary.geo_by_click_id[seed_click_id]
|
||||
return UserProfile(
|
||||
user_domain_id=device["user_domain_id"],
|
||||
seed_click_id=seed_click_id,
|
||||
device=device,
|
||||
geo=geo,
|
||||
)
|
||||
|
||||
def _rotate_new_user(self) -> UserProfile | None:
|
||||
inactive_users = [user for user in self.users if not user.is_active]
|
||||
if not inactive_users:
|
||||
return None
|
||||
|
||||
new_user = self._create_user()
|
||||
victim = min(
|
||||
inactive_users,
|
||||
key=lambda user: user.last_finished_at or datetime.min,
|
||||
)
|
||||
self.users[self.users.index(victim)] = new_user
|
||||
return new_user
|
||||
|
||||
def _is_available(self, user: UserProfile, tick_time: datetime) -> bool:
|
||||
if user.is_active:
|
||||
return False
|
||||
if user.last_finished_at is None:
|
||||
return True
|
||||
cooldown = timedelta(minutes=self.generator.config.min_return_minutes)
|
||||
return tick_time - user.last_finished_at >= cooldown
|
||||
|
||||
def _profile_seed_click_ids(self) -> list[str]:
|
||||
return [
|
||||
click_id
|
||||
for click_id in self.generator.dictionary.device_by_click_id
|
||||
if click_id in self.generator.dictionary.geo_by_click_id
|
||||
]
|
||||
|
||||
|
||||
def _empty_batch() -> dict[str, list[dict]]:
|
||||
return {topic: [] for topic in TOPICS}
|
||||
|
||||
@@ -44,8 +135,17 @@ class TickStreamGenerator:
|
||||
def __init__(self, generator: EventGenerator):
|
||||
self.generator = generator
|
||||
self.active_visits: list[ActiveVisit] = []
|
||||
self.population = UserPopulation(generator)
|
||||
self._pending_event_budget = 0
|
||||
|
||||
@property
|
||||
def population_size(self) -> int:
|
||||
return len(self.population.users)
|
||||
|
||||
@property
|
||||
def population_user_ids(self) -> set[str]:
|
||||
return {user.user_domain_id for user in self.population.users}
|
||||
|
||||
def generate_tick(
|
||||
self,
|
||||
event_budget: int,
|
||||
@@ -74,9 +174,15 @@ class TickStreamGenerator:
|
||||
self._pending_event_budget > 0
|
||||
and len(self.active_visits) < self.generator.config.max_active_sessions
|
||||
):
|
||||
user = self.population.choose_for_visit(tick_time)
|
||||
if user is None:
|
||||
self._pending_event_budget = 0
|
||||
return
|
||||
|
||||
visit_batch = self.generator.generate_batch(
|
||||
self.generator.config.max_session_events,
|
||||
planned_start_at=tick_time,
|
||||
user_profile={"device": user.device, "geo": user.geo},
|
||||
)
|
||||
timestamps = [
|
||||
_parse_timestamp(event["event_timestamp"])
|
||||
@@ -85,7 +191,11 @@ class TickStreamGenerator:
|
||||
if not timestamps:
|
||||
break
|
||||
|
||||
self.active_visits.append(ActiveVisit(batch=visit_batch, timestamps=timestamps))
|
||||
click_id = visit_batch["browser_events"][0]["click_id"]
|
||||
self.population.start_visit(user, click_id)
|
||||
self.active_visits.append(
|
||||
ActiveVisit(batch=visit_batch, timestamps=timestamps, user=user)
|
||||
)
|
||||
self._pending_event_budget -= len(timestamps)
|
||||
|
||||
if len(self.active_visits) >= self.generator.config.max_active_sessions:
|
||||
@@ -104,6 +214,10 @@ class TickStreamGenerator:
|
||||
visit.next_index += 1
|
||||
|
||||
def _drop_finished_visits(self) -> None:
|
||||
for visit in self.active_visits:
|
||||
if visit.is_finished:
|
||||
self.population.finish_visit(visit.user, visit.timestamps[-1])
|
||||
|
||||
self.active_visits = [
|
||||
visit
|
||||
for visit in self.active_visits
|
||||
|
||||
Reference in New Issue
Block a user