feat(generator): добавлено состояние версии 2 для рестартов

- Зачем:
  - генератор должен переживать рестарт без потери популяции пользователей и коротко прерванных активных визитов.
- Что:
  - добавлен компактный state v2 для популяции, активных визитов и остатка бюджета рождений.
  - сервис генератора переведён на единый тиковый поток с сохранением и восстановлением состояния.
  - добавлена безопасная деградация для старого state v1 и битого state v2.
  - покрыты короткий и долгий простой, reset состояния и валидация вложенного state.
- Проверка:
  - uv run --with-requirements generator/requirements.txt pytest generator/tests -q.
  - git diff --check.
This commit is contained in:
Dmitry Dementiev
2026-06-11 18:01:57 +03:00
parent 8f1e997cc9
commit 640050e409
7 changed files with 964 additions and 35 deletions
+22 -10
View File
@@ -23,8 +23,7 @@ from clickstream_generator.metrics import (
METRICS_LAST_SUCCESS,
METRICS_TICK_DURATION,
)
from clickstream_generator.runtime import generate_tick_batch
from clickstream_generator.state import GeneratorState
from clickstream_generator.runtime import TickStreamGenerator
logger = logging.getLogger("generator")
@@ -37,6 +36,7 @@ class GeneratorService:
self.config = config
self.dictionary = EventDictionary.load(config.data_dir)
self.generator = EventGenerator(self.dictionary, config)
self.stream = TickStreamGenerator(self.generator)
self.publisher: KafkaPublisher | None = None
self.history: KafkaBatchHistory | None = None
self.state_manager: KafkaStateManager | None = None
@@ -71,12 +71,24 @@ class GeneratorService:
if not self.config.state_reset:
restored_state = self.state_manager.load()
if restored_state:
self._tick = restored_state.tick
self.generator.rng.setstate(restored_state.rng_state)
logger.info(
f"Restored state: continuing from tick {self._tick}, "
f"last_batch_id={restored_state.last_batch_id}"
)
try:
self.generator.rng.setstate(restored_state.rng_state)
self.stream.restore_state(
restored_state,
restarted_at=datetime.now(timezone.utc),
)
self._tick = restored_state.tick
logger.info(
f"Restored state: continuing from tick {self._tick}, "
f"last_batch_id={restored_state.last_batch_id}"
)
except Exception as e:
logger.warning(
f"State data was invalid, starting fresh: {e}"
)
self.generator = EventGenerator(self.dictionary, self.config)
self.stream = TickStreamGenerator(self.generator)
self._tick = 0
else:
logger.info("State reset requested, starting fresh")
else:
@@ -108,7 +120,7 @@ class GeneratorService:
return
try:
state = GeneratorState(
state = self.stream.to_state(
tick=self._tick,
rng_state=self.generator.rng.getstate(),
last_batch_id=batch_id,
@@ -136,7 +148,7 @@ class GeneratorService:
logger.info(f"Generating with event budget ~{events_count}")
gen_start = time.time()
batch = generate_tick_batch(self.generator, events_count)
batch = self.stream.generate_tick(events_count)
gen_duration = time.time() - gen_start
pub_start = time.time()