refactor(generator): удалён in-memory fallback для истории батчей
- Удалён класс InMemoryBatchHistory и вся fallback-логика - Упрощён KafkaBatchHistory: убраны _initialized, get_stats(), обработка ошибок - Обновлена документация (generator/README.md, docs/OPERATIONS.md) - Упрощены тесты, удалены тесты для удалённого функционала - Код стал честнее: без Kafka генератор падает при старте Ревьюер: Prometheus даёт достаточно visibility, fallback избыточен
This commit is contained in:
Dmitry Dementiev
@@ -38,7 +38,6 @@ def base_config(data_dir):
|
||||
seed=42,
|
||||
enabled=True,
|
||||
metrics_port=9109,
|
||||
history_topic="generator_batch_history",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -58,7 +58,6 @@ class TestConfigDefaults:
|
||||
seed=None,
|
||||
enabled=True,
|
||||
metrics_port=9109,
|
||||
history_topic="generator_batch_history",
|
||||
)
|
||||
assert config.tick_seconds == 5
|
||||
finally:
|
||||
|
||||
@@ -1,89 +1,11 @@
|
||||
"""
|
||||
Тесты истории батчей.
|
||||
Тесты структуры записи батча.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from generator import InMemoryBatchHistory, BatchRecord
|
||||
|
||||
|
||||
class TestInMemoryBatchHistory:
|
||||
"""Тесты in-memory истории."""
|
||||
|
||||
def test_add_record(self):
|
||||
"""Добавление записи в историю."""
|
||||
history = InMemoryBatchHistory()
|
||||
record = BatchRecord(
|
||||
batch_id="test_1",
|
||||
started_at=datetime.now(timezone.utc),
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
sent_total=100,
|
||||
sent_browser=25,
|
||||
sent_location=25,
|
||||
sent_device=25,
|
||||
sent_geo=25,
|
||||
status="success",
|
||||
error_message=None,
|
||||
)
|
||||
history.add(record)
|
||||
|
||||
stats = history.get_stats()
|
||||
assert stats["total_batches"] == 1
|
||||
assert stats["success_rate"] == 1.0
|
||||
assert stats["last_batch_status"] == "success"
|
||||
|
||||
def test_history_limits_to_1000(self):
|
||||
"""История ограничена 1000 записями."""
|
||||
history = InMemoryBatchHistory()
|
||||
|
||||
for i in range(1100):
|
||||
record = BatchRecord(
|
||||
batch_id=f"batch_{i}",
|
||||
started_at=datetime.now(timezone.utc),
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
sent_total=10,
|
||||
sent_browser=2,
|
||||
sent_location=2,
|
||||
sent_device=2,
|
||||
sent_geo=2,
|
||||
status="success",
|
||||
error_message=None,
|
||||
)
|
||||
history.add(record)
|
||||
|
||||
assert len(history.batches) == 1000
|
||||
assert history.batches[0].batch_id == "batch_100" # Первые 100 удалены
|
||||
|
||||
def test_success_rate_calculation(self):
|
||||
"""Правильный расчет success rate."""
|
||||
history = InMemoryBatchHistory()
|
||||
|
||||
# 3 success, 2 error
|
||||
for i in range(5):
|
||||
record = BatchRecord(
|
||||
batch_id=f"batch_{i}",
|
||||
started_at=datetime.now(timezone.utc),
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
sent_total=100,
|
||||
sent_browser=25,
|
||||
sent_location=25,
|
||||
sent_device=25,
|
||||
sent_geo=25,
|
||||
status="success" if i < 3 else "error",
|
||||
error_message=None if i < 3 else "Test error",
|
||||
)
|
||||
history.add(record)
|
||||
|
||||
stats = history.get_stats()
|
||||
assert stats["total_batches"] == 5
|
||||
assert stats["success_rate"] == 0.6 # 3/5
|
||||
|
||||
def test_empty_history_returns_empty_stats(self):
|
||||
"""Пустая история возвращает пустой dict."""
|
||||
history = InMemoryBatchHistory()
|
||||
stats = history.get_stats()
|
||||
assert stats == {}
|
||||
from generator import BatchRecord
|
||||
|
||||
|
||||
class TestBatchRecord:
|
||||
|
||||
@@ -98,21 +98,17 @@ class TestKafkaBatchHistory:
|
||||
|
||||
history = KafkaBatchHistory("localhost:9092")
|
||||
|
||||
assert history._initialized is True
|
||||
assert history.bootstrap_servers == "localhost:9092"
|
||||
mock_producer_class.assert_called_once()
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_init_handles_connection_error(self, mock_import):
|
||||
"""Инициализация обрабатывает ошибку подключения."""
|
||||
# Симулируем ошибку при создании producer
|
||||
def test_init_raises_on_connection_error(self, mock_import):
|
||||
"""Инициализация падает при ошибке подключения."""
|
||||
mock_producer_class = MagicMock(side_effect=Exception("Connection failed"))
|
||||
mock_import.return_value = (mock_producer_class, Exception)
|
||||
|
||||
history = KafkaBatchHistory("localhost:9092")
|
||||
|
||||
assert history._initialized is False
|
||||
assert history.producer is None
|
||||
with pytest.raises(Exception, match="Connection failed"):
|
||||
KafkaBatchHistory("localhost:9092")
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_add_sends_to_kafka(self, mock_import):
|
||||
@@ -145,32 +141,8 @@ class TestKafkaBatchHistory:
|
||||
assert "value" in call_args[1]
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_add_skips_if_not_initialized(self, mock_import):
|
||||
"""add пропускает если не инициализирован."""
|
||||
mock_producer_class = MagicMock(side_effect=Exception("Connection failed"))
|
||||
mock_import.return_value = (mock_producer_class, Exception)
|
||||
|
||||
history = KafkaBatchHistory("localhost:9092")
|
||||
now = datetime.now(timezone.utc)
|
||||
record = BatchRecord(
|
||||
batch_id="skip456",
|
||||
started_at=now,
|
||||
finished_at=now,
|
||||
sent_total=0,
|
||||
sent_browser=0,
|
||||
sent_location=0,
|
||||
sent_device=0,
|
||||
sent_geo=0,
|
||||
status="error",
|
||||
error_message="Test",
|
||||
)
|
||||
|
||||
# Не должно упасть
|
||||
history.add(record)
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_add_handles_send_error(self, mock_import):
|
||||
"""add обрабатывает ошибку отправки."""
|
||||
def test_add_raises_on_send_error(self, mock_import):
|
||||
"""add пробрасывает ошибку отправки."""
|
||||
mock_producer = MagicMock()
|
||||
mock_producer.send.side_effect = Exception("Send failed")
|
||||
mock_producer_class = MagicMock(return_value=mock_producer)
|
||||
@@ -191,8 +163,8 @@ class TestKafkaBatchHistory:
|
||||
error_message=None,
|
||||
)
|
||||
|
||||
# Не должно упасть
|
||||
history.add(record)
|
||||
with pytest.raises(Exception, match="Send failed"):
|
||||
history.add(record)
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_flush_calls_producer_flush(self, mock_import):
|
||||
@@ -206,16 +178,6 @@ class TestKafkaBatchHistory:
|
||||
|
||||
mock_producer.flush.assert_called_once()
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_flush_noop_if_not_initialized(self, mock_import):
|
||||
"""flush ничего не делает если не инициализирован."""
|
||||
mock_producer_class = MagicMock(side_effect=Exception("Connection failed"))
|
||||
mock_import.return_value = (mock_producer_class, Exception)
|
||||
|
||||
history = KafkaBatchHistory("localhost:9092")
|
||||
# Не должно упасть
|
||||
history.flush()
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_close_calls_producer_close(self, mock_import):
|
||||
"""close вызывает close у producer."""
|
||||
@@ -227,37 +189,3 @@ class TestKafkaBatchHistory:
|
||||
history.close()
|
||||
|
||||
mock_producer.close.assert_called_once()
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_close_noop_if_not_initialized(self, mock_import):
|
||||
"""close ничего не делает если не инициализирован."""
|
||||
mock_producer_class = MagicMock(side_effect=Exception("Connection failed"))
|
||||
mock_import.return_value = (mock_producer_class, Exception)
|
||||
|
||||
history = KafkaBatchHistory("localhost:9092")
|
||||
# Не должно упасть
|
||||
history.close()
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_get_stats_returns_status(self, mock_import):
|
||||
"""get_stats возвращает статус инициализации."""
|
||||
mock_producer_class = MagicMock()
|
||||
mock_import.return_value = (mock_producer_class, Exception)
|
||||
|
||||
history = KafkaBatchHistory("localhost:9092")
|
||||
stats = history.get_stats()
|
||||
|
||||
assert stats["initialized"] is True
|
||||
assert stats["topic"] == "generator_batch_history"
|
||||
|
||||
@patch("generator._import_kafka")
|
||||
def test_get_stats_handles_not_initialized(self, mock_import):
|
||||
"""get_stats корректен при неинициализированном состоянии."""
|
||||
mock_producer_class = MagicMock(side_effect=Exception("Connection failed"))
|
||||
mock_import.return_value = (mock_producer_class, Exception)
|
||||
|
||||
history = KafkaBatchHistory("localhost:9092")
|
||||
stats = history.get_stats()
|
||||
|
||||
assert stats["initialized"] is False
|
||||
assert stats["topic"] == "generator_batch_history"
|
||||
|
||||
@@ -6,25 +6,10 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from generator import (
|
||||
Config, EventDictionary, GeneratorService,
|
||||
InMemoryBatchHistory, KafkaBatchHistory
|
||||
Config, EventDictionary, GeneratorService, KafkaBatchHistory
|
||||
)
|
||||
|
||||
|
||||
class TestConfigHistoryTopic:
|
||||
"""Тесты для history_topic в конфигурации."""
|
||||
|
||||
def test_default_history_topic(self, base_config):
|
||||
"""По умолчанию топик истории."""
|
||||
assert base_config.history_topic == "generator_batch_history"
|
||||
|
||||
def test_custom_history_topic(self, base_config):
|
||||
"""Кастомный топик истории."""
|
||||
from dataclasses import replace
|
||||
custom_config = replace(base_config, history_topic="custom_history")
|
||||
assert custom_config.history_topic == "custom_history"
|
||||
|
||||
|
||||
class TestBatchRecordWithDictConversion:
|
||||
"""Тесты конвертации BatchRecord в dict."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user