feat(diarization): подготовлен стенд Intel-эксперимента
Зачем: - нужен воспроизводимый Intel-этап перед проверкой кандидатов на Ryzen - sherpa-onnx 1.13.6 открывает настройку шага окна Что: - добавлены строгий manifest v3, планировщик A/B и проверяемая приватная капсула - реализованы экспериментальные NME, модели эмбеддингов и режимы worker/session - версия sherpa-onnx поднята до 1.13.6, зависимости и тесты закреплены Проверка: - uv lock --check - uv run pytest -q - uv run ruff check для изменённых Python-файлов - uv build с проверкой состава wheel
This commit is contained in:
Dmitry Dementiev committed
1 parent
7dc5723742
commit
bd0d7e6736
8 files changed
+5055
-62
No files matched your search
@@ -0,0 +1,923 @@
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from local_transcriber import benchmark_experiment as experiment
|
||||
|
||||
|
||||
def _write(path: Path, content: bytes) -> dict[str, object]:
|
||||
path.write_bytes(content)
|
||||
return {
|
||||
"path": str(path),
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"size_bytes": len(content),
|
||||
}
|
||||
|
||||
|
||||
def _artifact(tmp_path: Path, artifact_id: str, content: bytes) -> dict[str, object]:
|
||||
return {
|
||||
"id": artifact_id,
|
||||
"kind": "onnx-model",
|
||||
**_write(tmp_path / f"{artifact_id}.onnx", content),
|
||||
"source_url": f"https://example.invalid/{artifact_id}",
|
||||
"revision": "v1",
|
||||
"license": "Apache-2.0",
|
||||
}
|
||||
|
||||
|
||||
def _cell(name: str, position: int, role: str) -> dict[str, object]:
|
||||
return {
|
||||
"name": name,
|
||||
"stage": "intel",
|
||||
"phase": "window",
|
||||
"recording_ids": ["data-test", "t2-bdma", "yantar"],
|
||||
"window_shift_ratio": 0.1 if role == "baseline" else 0.2,
|
||||
"segmentation_artifact_id": "pyannote-fp32",
|
||||
"embedding_artifact_id": "wespeaker-fp32",
|
||||
"counter": "threshold",
|
||||
"clustering": {"mode": "threshold", "threshold": 0.89, "num_clusters": None},
|
||||
"inference": {
|
||||
"mode": "sequential",
|
||||
"outer_workers": 1,
|
||||
"session_count": 1,
|
||||
"intra_op_threads": 4,
|
||||
"inter_op_threads": 1,
|
||||
"batch_size": 1,
|
||||
},
|
||||
"repetition": 1,
|
||||
"schedule_position": position,
|
||||
"pair_id": "window-ab",
|
||||
"pair_role": role,
|
||||
"enabled": True,
|
||||
"skip_reason": None,
|
||||
"build_sha256": "9" * 64,
|
||||
}
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path) -> dict[str, object]:
|
||||
artifacts = [
|
||||
_artifact(tmp_path, "pyannote-fp32", b"segmentation"),
|
||||
_artifact(tmp_path, "wespeaker-fp32", b"wespeaker"),
|
||||
_artifact(tmp_path, "wespeaker-qdq", b"qdq"),
|
||||
]
|
||||
recordings = []
|
||||
for index, (recording_id, speakers) in enumerate(
|
||||
(("data-test", 3), ("t2-bdma", 2), ("yantar", 2))
|
||||
):
|
||||
media = _write(tmp_path / f"{recording_id}.wav", recording_id.encode())
|
||||
words = _write(
|
||||
tmp_path / f"{recording_id}.words.json",
|
||||
json.dumps([{"start": 0.0, "end": 0.5, "text": "слово"}]).encode(),
|
||||
)
|
||||
reference = None
|
||||
if index:
|
||||
side = _write(tmp_path / f"{recording_id}.md", b"reference")
|
||||
reference = {
|
||||
"path": side["path"],
|
||||
"sha256": side["sha256"],
|
||||
"format": "hypescribe_markdown",
|
||||
}
|
||||
recordings.append(
|
||||
{
|
||||
"id": recording_id,
|
||||
"role": "control",
|
||||
"path": media["path"],
|
||||
"sha256": media["sha256"],
|
||||
"start": 0.0,
|
||||
"duration": 300.0,
|
||||
"expected_speakers": speakers,
|
||||
"reference": reference,
|
||||
"asr_words": {
|
||||
"path": words["path"],
|
||||
"sha256": words["sha256"],
|
||||
"format": "local_transcriber_words_v1",
|
||||
},
|
||||
}
|
||||
)
|
||||
calibration = _write(tmp_path / "calibration.wav", b"separate calibration")
|
||||
return {
|
||||
"schema_version": 3,
|
||||
"experiment": {
|
||||
"id": "cpu-diarization-intel",
|
||||
"stage": "intel",
|
||||
"source_commit": "1" * 64,
|
||||
"handoff_commit": None,
|
||||
"sherpa_patch_sha256": "2" * 64,
|
||||
"python_version": "3.13.0",
|
||||
"dependencies": {
|
||||
"sherpa-onnx": "1.13.6",
|
||||
"sherpa-onnx-core": "1.13.6",
|
||||
"onnxruntime": "1.28.0",
|
||||
"numpy": "2.4.3",
|
||||
"onnx": "1.20.0",
|
||||
"kaldi-native-fbank": "1.22.3",
|
||||
"psutil": "7.0.0",
|
||||
},
|
||||
},
|
||||
"machine": {
|
||||
"id": "intel-i7-6820hq",
|
||||
"sku": "Intel Core i7-6820HQ",
|
||||
"physical_cores": 4,
|
||||
"logical_cores": os.cpu_count(),
|
||||
"ram_bytes": 32 * 1024**3,
|
||||
"os": "Linux test",
|
||||
"cpu_flags": ["avx", "avx2"],
|
||||
"power": {
|
||||
"supply": "ac",
|
||||
"profile": "balanced",
|
||||
"governor": "powersave",
|
||||
"turbo": True,
|
||||
"power_limits": "platform-default",
|
||||
"temperature_celsius": 55.0,
|
||||
"throttling": False,
|
||||
"swap_total_bytes": 8 * 1024**3,
|
||||
"swap_used_bytes": 0,
|
||||
},
|
||||
},
|
||||
"artifacts": artifacts,
|
||||
"recordings": recordings,
|
||||
"calibration": [
|
||||
{
|
||||
"id": "calibration-1",
|
||||
"path": calibration["path"],
|
||||
"sha256": calibration["sha256"],
|
||||
"source_id": "calibration-source-1",
|
||||
"source_sha256": calibration["sha256"],
|
||||
"source_start": 10.0,
|
||||
"source_duration": 30.0,
|
||||
"derived": False,
|
||||
}
|
||||
],
|
||||
"qdq": {
|
||||
"enabled": True,
|
||||
"source_ids": ["calibration-1"],
|
||||
"preprocessing": {"sample_rate": 16000},
|
||||
"reader": {"name": "speaker-features-v1"},
|
||||
"quant_pre_process": {"skip_optimization": False},
|
||||
"quantize_static": {
|
||||
"format": "QDQ",
|
||||
"activation": "QInt8",
|
||||
"weight": "QInt8",
|
||||
},
|
||||
"excluded_nodes": [],
|
||||
"tool_versions": {"onnxruntime": "1.28.0"},
|
||||
"graph_artifact_id": "wespeaker-qdq",
|
||||
"skip_reason": None,
|
||||
},
|
||||
"cells": [_cell("W0", 1, "baseline"), _cell("W2", 2, "candidate")],
|
||||
"handoff": {
|
||||
"outcome": "pending",
|
||||
"source_commit": None,
|
||||
"candidate_recipes": [],
|
||||
"public_report_sha256": None,
|
||||
"capsule_id": None,
|
||||
"capsule_sha256": None,
|
||||
},
|
||||
"rss_sample_interval_ms": 10,
|
||||
}
|
||||
|
||||
|
||||
def _environment() -> dict[str, object]:
|
||||
return {
|
||||
"packages": {
|
||||
"sherpa-onnx": "1.13.6",
|
||||
"sherpa-onnx-core": "1.13.6",
|
||||
"onnxruntime": "1.28.0",
|
||||
"numpy": "2.4.3",
|
||||
"onnx": "1.20.0",
|
||||
"kaldi-native-fbank": "1.22.3",
|
||||
"psutil": "7.0.0",
|
||||
},
|
||||
"python": "3.13.0",
|
||||
"platform": "Linux test",
|
||||
"logical_cores": os.cpu_count(),
|
||||
"physical_cores": 4,
|
||||
"cpu_sku": "Intel Core i7-6820HQ",
|
||||
"cpu_flags": ["avx", "avx2"],
|
||||
"ram_bytes": 32 * 1024**3,
|
||||
"swap_total_bytes": 8 * 1024**3,
|
||||
"swap_used_bytes": 0,
|
||||
"build_sha256": "9" * 64,
|
||||
}
|
||||
|
||||
|
||||
def _guard(**changes) -> dict[str, object]:
|
||||
result = {
|
||||
"supply": "ac",
|
||||
"profile": "balanced",
|
||||
"governor": "powersave",
|
||||
"turbo": True,
|
||||
"power_limits": {"package": "platform-default"},
|
||||
"temperature_celsius": 55.0,
|
||||
"throttling": False,
|
||||
"throttle_count": 10,
|
||||
"swap_used_bytes": 0,
|
||||
"swap_sin_bytes": 100,
|
||||
"swap_sout_bytes": 200,
|
||||
}
|
||||
result.update(changes)
|
||||
return result
|
||||
|
||||
|
||||
def test_manifest_identity_is_path_independent_and_covers_schedule(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
experiment.validate_manifest(manifest)
|
||||
first_id = experiment.make_cell_id(manifest, manifest["cells"][0])
|
||||
moved = copy.deepcopy(manifest)
|
||||
for artifact in moved["artifacts"]:
|
||||
artifact["path"] = "/private/moved/" + Path(artifact["path"]).name
|
||||
for recording in moved["recordings"]:
|
||||
recording["path"] = "/private/moved/recording"
|
||||
recording["asr_words"]["path"] = "/private/moved/words"
|
||||
if recording["reference"]:
|
||||
recording["reference"]["path"] = "/private/moved/reference"
|
||||
moved["calibration"][0]["path"] = "/private/moved/calibration"
|
||||
|
||||
assert experiment.make_cell_id(moved, moved["cells"][0]) == first_id
|
||||
|
||||
moved["cells"][0]["schedule_position"] = 3
|
||||
assert experiment.make_cell_id(moved, moved["cells"][0]) != first_id
|
||||
assert experiment.make_experiment_id(moved) != experiment.make_experiment_id(
|
||||
manifest
|
||||
)
|
||||
|
||||
finalized = copy.deepcopy(manifest)
|
||||
finalized["handoff"]["outcome"] = "stop-before-ryzen"
|
||||
assert experiment.make_experiment_id(finalized) == experiment.make_experiment_id(
|
||||
manifest
|
||||
)
|
||||
|
||||
|
||||
def test_cell_identity_covers_sidecars_and_qdq_recipe(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
cell = manifest["cells"][0]
|
||||
cell["embedding_artifact_id"] = "wespeaker-qdq"
|
||||
first_id = experiment.make_cell_id(manifest, cell)
|
||||
|
||||
changed_sidecar = copy.deepcopy(manifest)
|
||||
changed_sidecar["recordings"][0]["asr_words"]["sha256"] = "7" * 64
|
||||
assert experiment.make_cell_id(changed_sidecar, changed_sidecar["cells"][0]) != first_id
|
||||
|
||||
changed_recipe = copy.deepcopy(manifest)
|
||||
changed_recipe["qdq"]["quantize_static"]["weight"] = "QInt4"
|
||||
assert experiment.make_cell_id(changed_recipe, changed_recipe["cells"][0]) != first_id
|
||||
|
||||
|
||||
def test_identity_and_runtime_validation_cover_exact_versions_and_build(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
first_id = experiment.make_cell_id(manifest, manifest["cells"][0])
|
||||
|
||||
changed_python = copy.deepcopy(manifest)
|
||||
changed_python["experiment"]["python_version"] = "3.13.1"
|
||||
assert (
|
||||
experiment.make_cell_id(changed_python, changed_python["cells"][0])
|
||||
!= first_id
|
||||
)
|
||||
|
||||
changed_numpy = copy.deepcopy(manifest)
|
||||
changed_numpy["experiment"]["dependencies"]["numpy"] = "2.4.4"
|
||||
assert (
|
||||
experiment.make_cell_id(changed_numpy, changed_numpy["cells"][0])
|
||||
!= first_id
|
||||
)
|
||||
with pytest.raises(ValueError, match="numpy==2.4.4"):
|
||||
experiment.validate_environment(changed_numpy, _environment())
|
||||
|
||||
changed_build = copy.deepcopy(_environment())
|
||||
changed_build["build_sha256"] = "8" * 64
|
||||
with pytest.raises(ValueError, match="runtime-сборки"):
|
||||
experiment.validate_environment(manifest, changed_build)
|
||||
|
||||
|
||||
def test_manifest_rejects_stage_boundary_thread_budget_and_non_alternating_ab(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][0]["stage"] = "ryzen"
|
||||
with pytest.raises(ValueError, match="граница"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][0]["inference"]["outer_workers"] = 5
|
||||
with pytest.raises(ValueError, match="бюджет"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][1]["pair_role"] = "baseline"
|
||||
with pytest.raises(ValueError, match="baseline/candidate"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
manifest = _manifest(tmp_path)
|
||||
second_pair = []
|
||||
for source, name, position in (
|
||||
(manifest["cells"][0], "W0-repeat", 3),
|
||||
(manifest["cells"][1], "W2-repeat", 4),
|
||||
):
|
||||
cell = copy.deepcopy(source)
|
||||
cell.update(name=name, schedule_position=position, pair_id="window-ab-2")
|
||||
second_pair.append(cell)
|
||||
manifest["cells"].extend(second_pair)
|
||||
with pytest.raises(ValueError, match="начальная роль"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
|
||||
def test_ab_pair_requires_exactly_two_adjacent_schedule_positions(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][1]["schedule_position"] = 3
|
||||
with pytest.raises(ValueError, match="соседними"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
manifest = _manifest(tmp_path)
|
||||
for source, name, position in (
|
||||
(manifest["cells"][0], "W0-repeat", 3),
|
||||
(manifest["cells"][1], "W2-repeat", 4),
|
||||
):
|
||||
cell = copy.deepcopy(source)
|
||||
cell.update(name=name, schedule_position=position)
|
||||
manifest["cells"].append(cell)
|
||||
with pytest.raises(ValueError, match="ровно одна пара"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutate", "message"),
|
||||
[
|
||||
(lambda manifest: manifest["cells"][0]["inference"].update(outer_workers=1.5), "целое"),
|
||||
(lambda manifest: manifest["cells"][0].update(window_shift_ratio=0.3), "сетку"),
|
||||
(lambda manifest: manifest["cells"][0]["clustering"].update(threshold=None), "число"),
|
||||
(
|
||||
lambda manifest: manifest["cells"][0].update(
|
||||
recording_ids=["data-test", "data-test"]
|
||||
),
|
||||
"запись",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_manifest_rejects_fractional_or_ambiguous_cell_fields(
|
||||
tmp_path, mutate, message
|
||||
):
|
||||
manifest = _manifest(tmp_path)
|
||||
mutate(manifest)
|
||||
|
||||
with pytest.raises((TypeError, ValueError), match=message):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
|
||||
def test_manifest_rejects_enabled_qdq_without_graph(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["qdq"]["graph_artifact_id"] = None
|
||||
|
||||
with pytest.raises(ValueError, match="нужен граф"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("changes", "message"),
|
||||
[
|
||||
({"supply": "battery"}, "от сети"),
|
||||
({"profile": "performance"}, "balanced"),
|
||||
({"throttling": True}, "нестабильно"),
|
||||
],
|
||||
)
|
||||
def test_manifest_rejects_unstable_power_contract(tmp_path, changes, message):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["machine"]["power"].update(changes)
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
|
||||
def test_resolve_manifest_paths_rejects_symlink_escape(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
capsule = tmp_path / "capsule"
|
||||
capsule.mkdir()
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
(capsule / "escape").symlink_to(outside, target_is_directory=True)
|
||||
manifest["artifacts"][0]["path"] = "escape/model.onnx"
|
||||
|
||||
with pytest.raises(ValueError, match="выходит за корень"):
|
||||
experiment.resolve_manifest_paths(manifest, capsule)
|
||||
|
||||
|
||||
def test_raw_manifest_requires_capsule_relative_paths(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="относительным"):
|
||||
experiment.validate_manifest(
|
||||
manifest,
|
||||
verify_files=False,
|
||||
require_relative_paths=True,
|
||||
)
|
||||
|
||||
|
||||
def test_private_locations_inside_repository_must_be_git_ignored():
|
||||
repository_root = Path(experiment.__file__).parents[2]
|
||||
|
||||
with pytest.raises(ValueError, match="не игнорируется Git"):
|
||||
experiment._require_private_location(
|
||||
repository_root / "docs" / "private-manifest.json",
|
||||
"manifest",
|
||||
)
|
||||
|
||||
experiment._require_private_location(
|
||||
repository_root / "tmp" / "private-manifest.json",
|
||||
"manifest",
|
||||
)
|
||||
|
||||
|
||||
def test_qdq_rejects_control_origin_and_skips_missing_material_safely(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["calibration"][0]["source_sha256"] = manifest["recordings"][0]["sha256"]
|
||||
with pytest.raises(ValueError, match="контрольная запись"):
|
||||
experiment.validate_manifest(manifest)
|
||||
|
||||
manifest = _manifest(tmp_path)
|
||||
Path(manifest["calibration"][0]["path"]).unlink()
|
||||
warnings = experiment.validate_manifest(manifest)
|
||||
plan = experiment.safe_plan(manifest, warnings, _environment())
|
||||
|
||||
assert plan["qdq_available"] is False
|
||||
serialized = json.dumps(plan, ensure_ascii=False)
|
||||
assert str(tmp_path) not in serialized
|
||||
assert "separate calibration" not in serialized
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cluster_sizes, dimensions, seed, noise, expected",
|
||||
[((3, 17), 4, 35, 0.1, 2), ((2, 3, 15), 3, 1, 0.3, 3)],
|
||||
)
|
||||
def test_nme_and_eigengap_count_block_fixtures(
|
||||
cluster_sizes, dimensions, seed, noise, expected
|
||||
):
|
||||
embeddings = experiment.synthetic_block_embeddings(
|
||||
cluster_sizes, dimensions=dimensions, seed=seed, noise=noise
|
||||
)
|
||||
|
||||
assert experiment.estimate_eigengap(embeddings)["num_clusters"] == expected
|
||||
assert experiment.estimate_nme(embeddings)["num_clusters"] == expected
|
||||
|
||||
|
||||
def test_counter_selection_handles_mixed_model_outcomes():
|
||||
results = []
|
||||
for model_id, nme_counts, eigengap_counts, nme_time, eigengap_time in (
|
||||
("campplus", [3, 2, 2], [3, 2, 2], 12.0, 11.5),
|
||||
("titanet", [3, 2, 2], [3, 3, 2], 8.0, 3.0),
|
||||
("broken", [3, 3, 2], [3, 2, 3], 1.0, 1.0),
|
||||
):
|
||||
for counter, counts, wall in (
|
||||
("nme", nme_counts, nme_time),
|
||||
("eigengap", eigengap_counts, eigengap_time),
|
||||
):
|
||||
for num_clusters in counts:
|
||||
results.append(
|
||||
{
|
||||
"embedding_id": model_id,
|
||||
"counter": counter,
|
||||
"num_clusters": num_clusters,
|
||||
"wall_seconds": wall / 3,
|
||||
}
|
||||
)
|
||||
|
||||
assert experiment.select_counter(results) == {
|
||||
"broken": None,
|
||||
"campplus": "eigengap",
|
||||
"titanet": "nme",
|
||||
}
|
||||
|
||||
|
||||
def test_window_selection_and_recipe_assembly_cover_step_only_and_deduplication():
|
||||
results = [
|
||||
{
|
||||
"window_shift_ratio": 0.1,
|
||||
"wall_seconds": 100.0,
|
||||
"quality_passed": True,
|
||||
"work_counters_explained": True,
|
||||
},
|
||||
{
|
||||
"window_shift_ratio": 0.15,
|
||||
"wall_seconds": 88.0,
|
||||
"quality_passed": True,
|
||||
"work_counters_explained": True,
|
||||
},
|
||||
{
|
||||
"window_shift_ratio": 0.2,
|
||||
"wall_seconds": 85.0,
|
||||
"quality_passed": True,
|
||||
"work_counters_explained": True,
|
||||
},
|
||||
]
|
||||
assert experiment.select_working_shift(results) == 0.15
|
||||
assert experiment.assemble_candidate_recipes(0.15, []) == [
|
||||
{
|
||||
"id": "C0",
|
||||
"window_shift_ratio": 0.15,
|
||||
"embedding": "wespeaker-fp32",
|
||||
"inference": "sequential",
|
||||
}
|
||||
]
|
||||
|
||||
branches = [
|
||||
{
|
||||
"quality_passed": True,
|
||||
"memory_passed": True,
|
||||
"diagnostic_approved": True,
|
||||
"speedup": 0.2,
|
||||
"recipe": {"embedding": "wespeaker-qdq", "inference": "shared-session"},
|
||||
"isa_stack": True,
|
||||
"stack_order": 1,
|
||||
},
|
||||
{
|
||||
"quality_passed": True,
|
||||
"memory_passed": True,
|
||||
"diagnostic_approved": True,
|
||||
"speedup": 0.15,
|
||||
"recipe": {"embedding": "wespeaker-qdq", "inference": "shared-session"},
|
||||
"isa_stack": False,
|
||||
},
|
||||
]
|
||||
recipes = experiment.assemble_candidate_recipes(0.15, branches)
|
||||
assert len(recipes) <= 3
|
||||
assert len(
|
||||
{
|
||||
json.dumps({k: v for k, v in item.items() if k != "id"}, sort_keys=True)
|
||||
for item in recipes
|
||||
}
|
||||
) == len(recipes)
|
||||
|
||||
unapproved = copy.deepcopy(branches)
|
||||
unapproved[0]["diagnostic_approved"] = False
|
||||
unapproved[0]["speedup"] = 0.9
|
||||
assert all(
|
||||
item.get("embedding") != "wespeaker-qdq"
|
||||
for item in experiment.assemble_candidate_recipes(0.15, unapproved[:1])
|
||||
)
|
||||
|
||||
|
||||
def test_schedule_preserves_raw_repeats_and_resumes_only_success(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
output_path = tmp_path / "results.json"
|
||||
work_dir = tmp_path / "work"
|
||||
calls = []
|
||||
|
||||
def runner(request):
|
||||
calls.append(request["cell"]["name"])
|
||||
return {"raw": request["cell"]["repetition"]}
|
||||
|
||||
first = experiment.run_schedule(
|
||||
manifest,
|
||||
output_path,
|
||||
work_dir,
|
||||
runner,
|
||||
environment=_environment(),
|
||||
guard_reader=_guard,
|
||||
)
|
||||
resumed_environment = _environment()
|
||||
resumed_environment["swap_used_bytes"] = 4096
|
||||
second = experiment.run_schedule(
|
||||
manifest,
|
||||
output_path,
|
||||
work_dir,
|
||||
runner,
|
||||
environment=resumed_environment,
|
||||
guard_reader=_guard,
|
||||
)
|
||||
|
||||
assert calls == ["W0", "W2"]
|
||||
assert len(first["cells"]) == 2
|
||||
assert len(second["cells"]) == 2
|
||||
assert all(item["result"] == {"raw": 1} for item in second["cells"])
|
||||
assert all(item["guard"]["attempts"][0]["valid"] for item in second["cells"])
|
||||
|
||||
|
||||
def test_runtime_guard_retries_once_and_persists_transition_evidence(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][1]["enabled"] = False
|
||||
snapshots = iter(
|
||||
[
|
||||
_guard(),
|
||||
_guard(swap_sin_bytes=101),
|
||||
_guard(temperature_celsius=57.0),
|
||||
_guard(temperature_celsius=58.0),
|
||||
]
|
||||
)
|
||||
calls = []
|
||||
|
||||
output = experiment.run_schedule(
|
||||
manifest,
|
||||
tmp_path / "results.json",
|
||||
tmp_path / "work",
|
||||
lambda request: calls.append(request["cell"]["name"]) or {"raw": 1},
|
||||
environment=_environment(),
|
||||
guard_reader=lambda: next(snapshots),
|
||||
)
|
||||
|
||||
assert calls == ["W0", "W0"]
|
||||
attempts = output["cells"][0]["guard"]["attempts"]
|
||||
assert [item["valid"] for item in attempts] == [False, True]
|
||||
assert attempts[0]["reasons"] == ["swap_sin_bytes-changed"]
|
||||
assert attempts[0]["before"]["temperature_celsius"] == 55.0
|
||||
assert attempts[1]["after"]["temperature_celsius"] == 58.0
|
||||
|
||||
|
||||
def test_runtime_guard_rejects_second_invalid_transition(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][1]["enabled"] = False
|
||||
snapshots = iter(
|
||||
[
|
||||
_guard(),
|
||||
_guard(throttle_count=11),
|
||||
_guard(),
|
||||
_guard(swap_sout_bytes=201),
|
||||
]
|
||||
)
|
||||
output_path = tmp_path / "results.json"
|
||||
|
||||
with pytest.raises(RuntimeError, match="W0"):
|
||||
experiment.run_schedule(
|
||||
manifest,
|
||||
output_path,
|
||||
tmp_path / "work",
|
||||
lambda _request: {"raw": 1},
|
||||
environment=_environment(),
|
||||
guard_reader=lambda: next(snapshots),
|
||||
)
|
||||
|
||||
persisted = json.loads(output_path.read_text(encoding="utf-8"))["cells"][0]
|
||||
assert persisted["status"] == "failed"
|
||||
assert persisted["error_type"] == "RuntimeGuardError"
|
||||
assert len(persisted["guard"]["attempts"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("after", "reason"),
|
||||
[
|
||||
(_guard(governor="performance"), "governor-mismatch"),
|
||||
(_guard(turbo=False), "turbo-mismatch"),
|
||||
(_guard(throttling=True), "throttling"),
|
||||
(
|
||||
_guard(power_limits={"package": "changed"}),
|
||||
"power-limits-changed",
|
||||
),
|
||||
(_guard(swap_sout_bytes=201), "swap_sout_bytes-changed"),
|
||||
],
|
||||
)
|
||||
def test_runtime_guard_checks_stable_machine_conditions(tmp_path, after, reason):
|
||||
reasons = experiment.validate_runtime_guard_transition(
|
||||
_manifest(tmp_path), _guard(), after
|
||||
)
|
||||
|
||||
assert reason in reasons
|
||||
|
||||
|
||||
def test_failed_cell_is_persisted_without_private_exception_text(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
output_path = tmp_path / "results.json"
|
||||
|
||||
def runner(_request):
|
||||
raise RuntimeError(f"private path: {tmp_path / 'secret.wav'}")
|
||||
|
||||
with pytest.raises(RuntimeError, match="W0"):
|
||||
experiment.run_schedule(
|
||||
manifest,
|
||||
output_path,
|
||||
tmp_path / "work",
|
||||
runner,
|
||||
environment=_environment(),
|
||||
guard_reader=_guard,
|
||||
)
|
||||
|
||||
serialized = output_path.read_text(encoding="utf-8")
|
||||
assert str(tmp_path) not in serialized
|
||||
assert "secret.wav" not in serialized
|
||||
assert "RuntimeError" in serialized
|
||||
|
||||
|
||||
def test_residual_cluster_metrics_keep_time_and_word_units_separate():
|
||||
intervals = [
|
||||
{"speaker": 0, "start": 0.0, "end": 8.0},
|
||||
{"speaker": 1, "start": 8.0, "end": 10.0},
|
||||
{"speaker": 2, "start": 10.0, "end": 11.0},
|
||||
]
|
||||
words = [
|
||||
{"speaker": 0},
|
||||
{"speaker": 1},
|
||||
{"speaker": 2},
|
||||
{"speaker": 2},
|
||||
{"speaker": None},
|
||||
]
|
||||
|
||||
metrics = experiment.residual_cluster_metrics(intervals, words, 2)
|
||||
|
||||
assert metrics["residual_speech_time_share"] == pytest.approx(1 / 11)
|
||||
assert metrics["residual_assigned_word_share"] == pytest.approx(1 / 2)
|
||||
|
||||
|
||||
def test_capsule_verification_checks_hashes_commits_and_safe_paths(tmp_path):
|
||||
payload = b"private"
|
||||
payload_hash = hashlib.sha256(payload).hexdigest()
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_bytes = b'{"schema_version":3}\n'
|
||||
manifest_path.write_bytes(manifest_bytes)
|
||||
capsule = {
|
||||
"capsule_id": "opaque-41",
|
||||
"source_commit": "3" * 64,
|
||||
"public_report_sha256": "4" * 64,
|
||||
"files": {
|
||||
"manifest.json": hashlib.sha256(manifest_bytes).hexdigest(),
|
||||
"data/payload.bin": payload_hash,
|
||||
},
|
||||
}
|
||||
archive_path = tmp_path / "capsule.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||
archive.writestr("capsule.json", json.dumps(capsule))
|
||||
archive.writestr("manifest.json", manifest_bytes)
|
||||
archive.writestr("data/payload.bin", payload)
|
||||
archive_hash = experiment.file_sha256(archive_path)
|
||||
|
||||
result = experiment.verify_capsule(
|
||||
archive_path,
|
||||
archive_hash,
|
||||
expected_manifest_path=manifest_path,
|
||||
expected_source_commit="3" * 64,
|
||||
expected_report_sha256="4" * 64,
|
||||
expected_capsule_id="opaque-41",
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"capsule_id": "opaque-41",
|
||||
"source_commit": "3" * 64,
|
||||
"public_report_sha256": "4" * 64,
|
||||
"file_count": 2,
|
||||
"manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(),
|
||||
}
|
||||
|
||||
unrelated_manifest = tmp_path / "unrelated-manifest.json"
|
||||
unrelated_manifest.write_bytes(b'{"schema_version":3,"other":true}\n')
|
||||
with pytest.raises(ValueError, match="не совпал с manifest.json капсулы"):
|
||||
experiment.verify_capsule(
|
||||
archive_path,
|
||||
archive_hash,
|
||||
expected_manifest_path=unrelated_manifest,
|
||||
)
|
||||
|
||||
unsafe_path = tmp_path / "unsafe.zip"
|
||||
with zipfile.ZipFile(unsafe_path, "w") as archive:
|
||||
archive.writestr("capsule.json", json.dumps(capsule))
|
||||
archive.writestr("manifest.json", manifest_bytes)
|
||||
archive.writestr("../payload.bin", payload)
|
||||
with pytest.raises(ValueError, match="небезопасный путь"):
|
||||
experiment.verify_capsule(
|
||||
unsafe_path,
|
||||
experiment.file_sha256(unsafe_path),
|
||||
expected_manifest_path=manifest_path,
|
||||
)
|
||||
|
||||
|
||||
def test_capsule_binds_candidate_result_evidence_to_file_index(tmp_path):
|
||||
result_bytes = b'{"aggregate":{"quality_passed":true}}'
|
||||
result_hash = hashlib.sha256(result_bytes).hexdigest()
|
||||
manifest_value = {
|
||||
"handoff": {
|
||||
"candidate_recipes": [
|
||||
{
|
||||
"result_file": "results/candidate.json",
|
||||
"result_sha256": result_hash,
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_bytes = json.dumps(manifest_value, separators=(",", ":")).encode()
|
||||
manifest_path.write_bytes(manifest_bytes)
|
||||
capsule = {
|
||||
"capsule_id": "opaque-41",
|
||||
"source_commit": "3" * 64,
|
||||
"public_report_sha256": "4" * 64,
|
||||
"files": {
|
||||
"manifest.json": hashlib.sha256(manifest_bytes).hexdigest(),
|
||||
"results/candidate.json": result_hash,
|
||||
},
|
||||
}
|
||||
archive_path = tmp_path / "capsule.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||
archive.writestr("capsule.json", json.dumps(capsule))
|
||||
archive.writestr("manifest.json", manifest_bytes)
|
||||
archive.writestr("results/candidate.json", result_bytes)
|
||||
|
||||
experiment.verify_capsule(
|
||||
archive_path,
|
||||
experiment.file_sha256(archive_path),
|
||||
expected_manifest_path=manifest_path,
|
||||
)
|
||||
|
||||
broken_manifest_value = copy.deepcopy(manifest_value)
|
||||
broken_manifest_value["handoff"]["candidate_recipes"][0]["result_sha256"] = (
|
||||
"5" * 64
|
||||
)
|
||||
broken_manifest_bytes = json.dumps(
|
||||
broken_manifest_value, separators=(",", ":")
|
||||
).encode()
|
||||
broken_manifest_path = tmp_path / "broken-manifest.json"
|
||||
broken_manifest_path.write_bytes(broken_manifest_bytes)
|
||||
capsule["files"]["manifest.json"] = hashlib.sha256(
|
||||
broken_manifest_bytes
|
||||
).hexdigest()
|
||||
broken_path = tmp_path / "broken-capsule.zip"
|
||||
with zipfile.ZipFile(broken_path, "w") as archive:
|
||||
archive.writestr("capsule.json", json.dumps(capsule))
|
||||
archive.writestr("manifest.json", broken_manifest_bytes)
|
||||
archive.writestr("results/candidate.json", result_bytes)
|
||||
with pytest.raises(ValueError, match="Результат кандидата"):
|
||||
experiment.verify_capsule(
|
||||
broken_path,
|
||||
experiment.file_sha256(broken_path),
|
||||
expected_manifest_path=broken_manifest_path,
|
||||
)
|
||||
|
||||
|
||||
def test_handoff_enforces_candidate_limit_and_stop_contract(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["handoff"]["outcome"] = "stop-before-ryzen"
|
||||
manifest["handoff"]["candidate_recipes"] = ["C0"]
|
||||
with pytest.raises(ValueError, match="ноль кандидатов"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["handoff"]["outcome"] = "handoff"
|
||||
manifest["handoff"]["candidate_recipes"] = ["C0", "C1", "C2", "C3"]
|
||||
with pytest.raises(ValueError, match="трех"):
|
||||
experiment.validate_manifest(manifest, verify_files=False)
|
||||
|
||||
|
||||
def test_finalize_handoff_binds_candidate_result_and_all_gates(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][0]["phase"] = "combination"
|
||||
cell_id = experiment.make_cell_id(manifest, manifest["cells"][0])
|
||||
payload = {"aggregate": {"quality_passed": True}, "raw": [1, 2, 3]}
|
||||
output = {
|
||||
"experiment_id": experiment.make_experiment_id(manifest),
|
||||
"cells": [{"cell_id": cell_id, "status": "complete", "result": payload}],
|
||||
}
|
||||
|
||||
finalized = experiment.finalize_handoff(
|
||||
manifest,
|
||||
output,
|
||||
["W0"],
|
||||
capsule_root=tmp_path / "capsule",
|
||||
gate_evidence={
|
||||
"W0": {"memory_passed": True, "diagnostic_approved": True}
|
||||
},
|
||||
public_report_sha256="4" * 64,
|
||||
capsule_id="opaque-41",
|
||||
)
|
||||
|
||||
candidate = finalized["handoff"]["candidate_recipes"][0]
|
||||
assert candidate == {
|
||||
"name": "W0",
|
||||
"cell_id": cell_id,
|
||||
"result_file": f"results/{cell_id}.json",
|
||||
"result_sha256": experiment._digest_json(payload),
|
||||
"mandatory_passed": True,
|
||||
"memory_passed": True,
|
||||
"diagnostic_approved": True,
|
||||
}
|
||||
result_file = tmp_path / "capsule" / candidate["result_file"]
|
||||
assert experiment.file_sha256(result_file) == candidate["result_sha256"]
|
||||
assert json.loads(result_file.read_text(encoding="utf-8")) == payload
|
||||
experiment.validate_manifest(finalized, verify_files=False)
|
||||
|
||||
finalized["handoff"]["capsule_sha256"] = "5" * 64
|
||||
experiment.validate_manifest(finalized, verify_files=False)
|
||||
|
||||
|
||||
def test_finalize_handoff_rejects_missing_diagnostic_approval(tmp_path):
|
||||
manifest = _manifest(tmp_path)
|
||||
manifest["cells"][0]["phase"] = "combination"
|
||||
cell_id = experiment.make_cell_id(manifest, manifest["cells"][0])
|
||||
output = {
|
||||
"experiment_id": experiment.make_experiment_id(manifest),
|
||||
"cells": [
|
||||
{
|
||||
"cell_id": cell_id,
|
||||
"status": "complete",
|
||||
"result": {"aggregate": {"quality_passed": True}},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="обязательные gates"):
|
||||
experiment.finalize_handoff(
|
||||
manifest,
|
||||
output,
|
||||
["W0"],
|
||||
capsule_root=tmp_path / "capsule",
|
||||
gate_evidence={
|
||||
"W0": {"memory_passed": True, "diagnostic_approved": False}
|
||||
},
|
||||
public_report_sha256="4" * 64,
|
||||
capsule_id="opaque-41",
|
||||
)
|
||||
@@ -137,7 +137,7 @@ def _manifest(tmp_path):
|
||||
|
||||
def _environment():
|
||||
return {
|
||||
"sherpa_onnx_version": "1.13.5",
|
||||
"sherpa_onnx_version": "1.13.6",
|
||||
"onnxruntime_version": "1",
|
||||
"numpy_version": "2",
|
||||
"psutil_version": "7",
|
||||
@@ -408,7 +408,7 @@ def test_stage_timing_parser_accepts_native_lines():
|
||||
"[I] OfflineSpeakerDiarization: clustering 0.100 s\n"
|
||||
"[I] OfflineSpeakerDiarization: total 3.100 s, audio 10.000 s, RTF 0.310"
|
||||
)
|
||||
stages = calibration.parse_stage_timings(log, "1.13.5")
|
||||
stages = calibration.parse_stage_timings(log, "1.13.6")
|
||||
assert stages["engine_total_seconds"] == 3.1
|
||||
assert stages["engine_audio_seconds"] == 10.0
|
||||
assert stages["engine_rtf"] == 0.31
|
||||
@@ -424,7 +424,7 @@ def test_stage_timing_parser_accepts_native_lines():
|
||||
)
|
||||
def test_stage_timing_parser_rejects_missing_duplicate_and_negative(log):
|
||||
with pytest.raises(ValueError):
|
||||
calibration.parse_stage_timings(log, "1.13.5")
|
||||
calibration.parse_stage_timings(log, "1.13.6")
|
||||
|
||||
|
||||
def test_stage_timing_parser_rejects_extra_malformed_record():
|
||||
@@ -436,7 +436,7 @@ def test_stage_timing_parser_rejects_extra_malformed_record():
|
||||
"OfflineSpeakerDiarization: total 3.100 s, audio 10.000 s, RTF 0.310"
|
||||
)
|
||||
with pytest.raises(ValueError, match="повреждённую"):
|
||||
calibration.parse_stage_timings(log, "1.13.5")
|
||||
calibration.parse_stage_timings(log, "1.13.6")
|
||||
|
||||
|
||||
def test_measurement_uses_injected_cpu_clock_and_rss_once():
|
||||
@@ -708,7 +708,7 @@ def test_execute_cell_rejects_text_mismatch_without_persisting_private_text(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
manifest = _manifest(tmp_path)
|
||||
output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.5"}}
|
||||
output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.6"}}
|
||||
spec = calibration.expand_automatic_cells(manifest, "e" * 64)[0]
|
||||
recording = manifest["recordings"][0]
|
||||
private_text = " СОВЕРШЕННО-СЕКРЕТНЫЙ-ТЕКСТ"
|
||||
@@ -758,7 +758,7 @@ def test_execute_cell_rejects_text_mismatch_without_persisting_private_text(
|
||||
|
||||
def test_execute_cell_rejects_engine_audio_duration_mismatch(tmp_path, monkeypatch):
|
||||
manifest = _manifest(tmp_path)
|
||||
output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.5"}}
|
||||
output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.6"}}
|
||||
spec = calibration.expand_automatic_cells(manifest, "e" * 64)[0]
|
||||
recording = manifest["recordings"][0]
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -0,0 +1,677 @@
|
||||
import io
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from local_transcriber import experimental_diarization as runner
|
||||
|
||||
|
||||
def _run_shared_cell(monkeypatch, tmp_path, *, counters=None, drop_output_text=False):
|
||||
events = []
|
||||
active_interval = {"name": None}
|
||||
embeddings = np.asarray([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32)
|
||||
work_counters = (
|
||||
{
|
||||
"embedding_jobs": 2,
|
||||
"embedding_run_calls": 2,
|
||||
"embedding_rows_dropped": 0,
|
||||
}
|
||||
if counters is None
|
||||
else counters
|
||||
)
|
||||
|
||||
class Diarization:
|
||||
def sort_by_start_time(self):
|
||||
return [
|
||||
SimpleNamespace(speaker=0, start=0.0, end=0.5),
|
||||
SimpleNamespace(speaker=1, start=0.5, end=1.0),
|
||||
]
|
||||
|
||||
class Engine:
|
||||
sample_rate = runner.SAMPLE_RATE
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
def prepare(self, samples, workers):
|
||||
events.append((active_interval["name"], f"prepare:{workers}"))
|
||||
return SimpleNamespace(
|
||||
is_terminal=False,
|
||||
embeddings=embeddings.copy(),
|
||||
row_mapping=((0, 0, [(0, 8_000)]), (1, 0, [(8_000, 16_000)])),
|
||||
counters=dict(work_counters),
|
||||
)
|
||||
|
||||
def finalize(self, prepared, labels):
|
||||
events.append((active_interval["name"], "finalize"))
|
||||
return Diarization()
|
||||
|
||||
class Clustering:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
def __call__(self, values):
|
||||
events.append((active_interval["name"], "cluster"))
|
||||
return np.asarray([0, 1], dtype=np.int32)
|
||||
|
||||
fake_sherpa = SimpleNamespace(
|
||||
OfflineSpeakerDiarization=Engine,
|
||||
FastClusteringConfig=lambda **values: SimpleNamespace(**values),
|
||||
FastClustering=Clustering,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "sherpa_onnx", fake_sherpa)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"make_config",
|
||||
lambda cell, artifacts, provider_config_path: SimpleNamespace(
|
||||
validate=lambda: True
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"_decode_clip",
|
||||
lambda recording: np.zeros(runner.SAMPLE_RATE, dtype=np.float32),
|
||||
)
|
||||
|
||||
def estimate(values):
|
||||
events.append((active_interval["name"], "counter"))
|
||||
return {"status": "complete", "num_clusters": 2, "p": 3}
|
||||
|
||||
monkeypatch.setattr(runner, "estimate_nme", estimate)
|
||||
measurement_names = iter(("published", "validation"))
|
||||
|
||||
def measure(operation, process, interval_seconds):
|
||||
name = next(measurement_names)
|
||||
active_interval["name"] = name
|
||||
try:
|
||||
result = operation()
|
||||
finally:
|
||||
active_interval["name"] = None
|
||||
suffix = 0 if name == "published" else 1
|
||||
return result, {
|
||||
"wall_seconds": 10.0 + suffix,
|
||||
"user_cpu_seconds": 7.0 + suffix,
|
||||
"system_cpu_seconds": 1.0,
|
||||
"total_cpu_seconds": 8.0 + suffix,
|
||||
"average_cpu_cores": 0.8,
|
||||
"peak_rss_bytes": 100 + suffix,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(runner, "measure_call", measure)
|
||||
|
||||
@contextmanager
|
||||
def native_log():
|
||||
yield io.BytesIO()
|
||||
|
||||
monkeypatch.setattr(runner, "capture_native_stderr", native_log)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"parse_prepare_timings",
|
||||
lambda log: {"segmentation_seconds": 0.2, "embedding_seconds": 0.1},
|
||||
)
|
||||
if drop_output_text:
|
||||
original_builder = runner.build_speaker_transcript
|
||||
|
||||
def build_without_last_word(words, intervals, duration):
|
||||
transcript = original_builder(words, intervals, duration)
|
||||
transcript.turns = transcript.turns[:-1]
|
||||
return transcript
|
||||
|
||||
monkeypatch.setattr(
|
||||
runner, "build_speaker_transcript", build_without_last_word
|
||||
)
|
||||
|
||||
words_path = tmp_path / "words.json"
|
||||
words_path.write_text(
|
||||
'[{"start": 0.1, "end": 0.4, "text": "Привет"}, '
|
||||
'{"start": 0.6, "end": 0.9, "text": "мир"}]',
|
||||
encoding="utf-8",
|
||||
)
|
||||
cell = {
|
||||
"name": "E1",
|
||||
"cell_id": "shared-cell",
|
||||
"counter": "nme",
|
||||
"window_shift_ratio": 0.1,
|
||||
"segmentation_artifact_id": "segmentation",
|
||||
"embedding_artifact_id": "embedding",
|
||||
"recording_ids": ["recording"],
|
||||
"clustering": {"threshold": 0.89, "num_clusters": None},
|
||||
"inference": {
|
||||
"mode": "shared-session",
|
||||
"outer_workers": 2,
|
||||
"session_count": 1,
|
||||
"intra_op_threads": 2,
|
||||
"inter_op_threads": 1,
|
||||
"batch_size": 1,
|
||||
},
|
||||
}
|
||||
request = {
|
||||
"manifest": {
|
||||
"rss_sample_interval_ms": 10,
|
||||
"artifacts": [],
|
||||
"recordings": [
|
||||
{
|
||||
"id": "recording",
|
||||
"path": str(tmp_path / "recording.wav"),
|
||||
"start": 0.0,
|
||||
"duration": 1.0,
|
||||
"expected_speakers": 2,
|
||||
"asr_words": {"path": str(words_path)},
|
||||
"reference": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
"cell": cell,
|
||||
"work_dir": str(tmp_path),
|
||||
}
|
||||
return runner.run_cell(request), events
|
||||
|
||||
|
||||
def test_run_cell_shared_session_separates_measurement_from_validation(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
result, events = _run_shared_cell(monkeypatch, tmp_path)
|
||||
|
||||
recording = result["recordings"][0]
|
||||
assert result["aggregate"]["quality_passed"] is True
|
||||
assert recording["text_equal"] is True
|
||||
assert recording["prepared"]["inference"] == {
|
||||
"mode": "shared-session",
|
||||
"rows_equal_to_sequential": True,
|
||||
"finite_embeddings": True,
|
||||
"labels_equal_to_sequential": True,
|
||||
"final_equal_to_sequential": True,
|
||||
"max_abs_drift": 0.0,
|
||||
"min_cosine": 1.0,
|
||||
}
|
||||
assert recording["metrics"]["wall_seconds"] == 10.0
|
||||
assert recording["metrics"]["peak_rss_bytes"] == 100
|
||||
assert recording["metrics"]["validation_wall_seconds"] == 11.0
|
||||
assert recording["metrics"]["validation_peak_rss_bytes"] == 101
|
||||
assert events == [
|
||||
("published", "prepare:2"),
|
||||
("published", "counter"),
|
||||
("published", "cluster"),
|
||||
("published", "finalize"),
|
||||
("validation", "counter"),
|
||||
("validation", "cluster"),
|
||||
("validation", "prepare:1"),
|
||||
("validation", "counter"),
|
||||
("validation", "cluster"),
|
||||
("validation", "finalize"),
|
||||
]
|
||||
|
||||
|
||||
def test_run_cell_rejects_changed_output_text(monkeypatch, tmp_path):
|
||||
result, _ = _run_shared_cell(
|
||||
monkeypatch, tmp_path, drop_output_text=True
|
||||
)
|
||||
|
||||
assert result["recordings"][0]["text_equal"] is False
|
||||
assert result["aggregate"]["quality_passed"] is False
|
||||
|
||||
|
||||
def test_run_cell_rejects_missing_prepared_work_counters(monkeypatch, tmp_path):
|
||||
result, _ = _run_shared_cell(monkeypatch, tmp_path, counters={})
|
||||
|
||||
assert result["recordings"][0]["prepared"]["validation_passed"] is False
|
||||
assert result["aggregate"]["quality_passed"] is False
|
||||
|
||||
|
||||
def test_segmentation_window_count_includes_last_padded_window():
|
||||
assert runner.segmentation_window_count(160_000, 16_000) == 1
|
||||
assert runner.segmentation_window_count(160_001, 16_000) == 2
|
||||
assert runner.segmentation_window_count(176_000, 16_000) == 2
|
||||
|
||||
|
||||
def test_prepare_timing_parser_accepts_exactly_two_stages():
|
||||
log = (
|
||||
"OfflineSpeakerDiarization: segmentation 1.250 s\n"
|
||||
"OfflineSpeakerDiarization: embedding 2.500 s"
|
||||
)
|
||||
|
||||
assert runner.parse_prepare_timings(log) == {
|
||||
"segmentation_seconds": 1.25,
|
||||
"embedding_seconds": 2.5,
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="обе стадии"):
|
||||
runner.parse_prepare_timings("OfflineSpeakerDiarization: embedding 2.5 s")
|
||||
|
||||
|
||||
def test_known_counter_clusters_independent_embedding_copies():
|
||||
original = np.asarray([[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]], dtype=np.float32)
|
||||
prepared = SimpleNamespace(embeddings=original, is_terminal=False)
|
||||
|
||||
class FakeClustering:
|
||||
calls = 0
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
def __call__(self, features):
|
||||
FakeClustering.calls += 1
|
||||
features[:] = -100
|
||||
return list(range(self.config.num_clusters)) + [0]
|
||||
|
||||
fake_sherpa = SimpleNamespace(
|
||||
FastClusteringConfig=lambda **values: SimpleNamespace(**values),
|
||||
FastClustering=FakeClustering,
|
||||
)
|
||||
cell = {
|
||||
"counter": "known",
|
||||
"clustering": {"threshold": 0.89},
|
||||
}
|
||||
|
||||
labels, counter, _ = runner._cluster_prepared(
|
||||
fake_sherpa, prepared, cell, expected_speakers=2
|
||||
)
|
||||
|
||||
assert labels.tolist() == [0, 1, 0]
|
||||
assert counter["num_clusters"] == 2
|
||||
assert FakeClustering.calls == 1
|
||||
validated = runner._validate_cluster_repeatability(
|
||||
fake_sherpa,
|
||||
prepared,
|
||||
cell,
|
||||
expected_speakers=2,
|
||||
labels=labels,
|
||||
counter_result=counter,
|
||||
)
|
||||
assert validated["repeatable_labels"] is True
|
||||
assert FakeClustering.calls == 2
|
||||
assert np.array_equal(prepared.embeddings, original)
|
||||
|
||||
|
||||
def _special_cell(mode, *, batch_size=1, sessions=1, outer=1, intra=4):
|
||||
return {
|
||||
"counter": "nme" if mode == "titanet-batch" else "known",
|
||||
"clustering": {"threshold": 0.89},
|
||||
"embedding_artifact_id": "embedding",
|
||||
"inference": {
|
||||
"mode": mode,
|
||||
"outer_workers": outer,
|
||||
"session_count": sessions,
|
||||
"intra_op_threads": intra,
|
||||
"inter_op_threads": 1,
|
||||
"batch_size": batch_size,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_special_inference_enforces_exact_thread_budgets():
|
||||
runner._validate_special_inference(
|
||||
_special_cell("separate-session", sessions=4, outer=4, intra=1), 4
|
||||
)
|
||||
runner._validate_special_inference(
|
||||
_special_cell("titanet-batch", batch_size=4), 4
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="P worker"):
|
||||
runner._validate_special_inference(
|
||||
_special_cell("separate-session", sessions=2, outer=4, intra=1), 4
|
||||
)
|
||||
with pytest.raises(ValueError, match="бюджет физических ядер"):
|
||||
runner._validate_special_inference(
|
||||
_special_cell("separate-session", sessions=4, outer=4, intra=2), 4
|
||||
)
|
||||
with pytest.raises(ValueError, match="batch 1 или 4"):
|
||||
runner._validate_special_inference(
|
||||
_special_cell("titanet-batch", batch_size=2), 4
|
||||
)
|
||||
|
||||
|
||||
def test_separate_sessions_restore_row_order_without_loss():
|
||||
class Stream:
|
||||
def __init__(self):
|
||||
self.samples = []
|
||||
|
||||
def accept_waveform(self, sample_rate, samples):
|
||||
assert sample_rate == runner.SAMPLE_RATE
|
||||
self.samples.extend(samples)
|
||||
|
||||
def input_finished(self):
|
||||
pass
|
||||
|
||||
class Extractor:
|
||||
def create_stream(self):
|
||||
return Stream()
|
||||
|
||||
def is_ready(self, stream):
|
||||
return bool(stream.samples)
|
||||
|
||||
def compute(self, stream):
|
||||
return [stream.samples[0], len(stream.samples)]
|
||||
|
||||
samples = np.arange(30, dtype=np.float32)
|
||||
mapping = [
|
||||
(0, 0, [(9, 12)]),
|
||||
(0, 1, [(1, 3)]),
|
||||
(1, 0, [(20, 24)]),
|
||||
(1, 1, [(5, 6)]),
|
||||
(2, 0, [(14, 19)]),
|
||||
]
|
||||
|
||||
embeddings = runner._compute_separate_embeddings(
|
||||
[Extractor(), Extractor()], mapping, samples, outer_workers=2
|
||||
)
|
||||
|
||||
assert embeddings.tolist() == [
|
||||
[9.0, 3.0],
|
||||
[1.0, 2.0],
|
||||
[20.0, 4.0],
|
||||
[5.0, 1.0],
|
||||
[14.0, 5.0],
|
||||
]
|
||||
assert np.isfinite(embeddings).all()
|
||||
|
||||
|
||||
def test_separate_runner_measures_validation_and_embedding_rss_separately(
|
||||
monkeypatch,
|
||||
):
|
||||
embeddings = np.asarray([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32)
|
||||
prepared = SimpleNamespace(
|
||||
is_terminal=False,
|
||||
embeddings=embeddings.copy(),
|
||||
row_mapping=((0, 0, [(0, 5)]), (1, 0, [(5, 10)])),
|
||||
counters={
|
||||
"embedding_jobs": 2,
|
||||
"embedding_run_calls": 2,
|
||||
"embedding_rows_dropped": 0,
|
||||
},
|
||||
)
|
||||
events = []
|
||||
active_interval = {"name": None}
|
||||
|
||||
class Clustering:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
def __call__(self, values):
|
||||
events.append((active_interval["name"], "cluster"))
|
||||
return np.asarray([0, 1], dtype=np.int32)
|
||||
|
||||
fake_sherpa = SimpleNamespace(
|
||||
FastClusteringConfig=lambda **values: SimpleNamespace(**values),
|
||||
FastClustering=Clustering,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "sherpa_onnx", fake_sherpa)
|
||||
|
||||
def estimate(values):
|
||||
events.append((active_interval["name"], "counter"))
|
||||
return {"status": "complete", "num_clusters": 2, "p": 3}
|
||||
|
||||
monkeypatch.setattr(runner, "estimate_nme", estimate)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"_compute_separate_embeddings",
|
||||
lambda extractors, mapping, samples, outer_workers: embeddings.copy(),
|
||||
)
|
||||
measurement_names = iter(("published", "embedding", "validation"))
|
||||
|
||||
def measure(operation, process, interval_seconds):
|
||||
name = next(measurement_names)
|
||||
previous = active_interval["name"]
|
||||
active_interval["name"] = name
|
||||
try:
|
||||
result = operation()
|
||||
finally:
|
||||
active_interval["name"] = previous
|
||||
values = {
|
||||
"published": (10.0, 100),
|
||||
"embedding": (3.0, 50),
|
||||
"validation": (4.0, 200),
|
||||
}
|
||||
wall, peak = values[name]
|
||||
return result, {
|
||||
"wall_seconds": wall,
|
||||
"user_cpu_seconds": wall / 2,
|
||||
"system_cpu_seconds": 1.0,
|
||||
"total_cpu_seconds": wall / 2 + 1.0,
|
||||
"average_cpu_cores": 0.6,
|
||||
"peak_rss_bytes": peak,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(runner, "measure_call", measure)
|
||||
|
||||
@contextmanager
|
||||
def native_log():
|
||||
yield io.BytesIO()
|
||||
|
||||
monkeypatch.setattr(runner, "capture_native_stderr", native_log)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"parse_prepare_timings",
|
||||
lambda log: {"segmentation_seconds": 0.2, "embedding_seconds": 0.1},
|
||||
)
|
||||
|
||||
class Diarization:
|
||||
def sort_by_start_time(self):
|
||||
return [
|
||||
SimpleNamespace(speaker=0, start=0.0, end=0.5),
|
||||
SimpleNamespace(speaker=1, start=0.5, end=1.0),
|
||||
]
|
||||
|
||||
class Engine:
|
||||
def prepare(self, samples, workers):
|
||||
events.append((active_interval["name"], "prepare"))
|
||||
return prepared
|
||||
|
||||
def finalize(self, prepared_value, labels):
|
||||
events.append((active_interval["name"], "finalize"))
|
||||
return Diarization()
|
||||
|
||||
cell = _special_cell("separate-session", sessions=2, outer=2, intra=1)
|
||||
cell["counter"] = "nme"
|
||||
_, metrics, _, data = runner._run_separate_session_recording(
|
||||
Engine(),
|
||||
[object(), object()],
|
||||
np.zeros(10, dtype=np.float32),
|
||||
cell,
|
||||
{"expected_speakers": 2},
|
||||
0.01,
|
||||
)
|
||||
|
||||
assert metrics["peak_rss_bytes"] == 100
|
||||
assert metrics["embedding_peak_rss_bytes"] == 50
|
||||
assert metrics["validation_peak_rss_bytes"] == 200
|
||||
assert metrics["validation_wall_seconds"] == 4.0
|
||||
assert metrics["instrumented_pipeline_wall_seconds"] == 10.0
|
||||
assert "measured_validation_wall_seconds" not in metrics
|
||||
assert data["validation_passed"] is True
|
||||
assert events == [
|
||||
("published", "prepare"),
|
||||
("published", "counter"),
|
||||
("published", "cluster"),
|
||||
("published", "finalize"),
|
||||
("validation", "counter"),
|
||||
("validation", "cluster"),
|
||||
("validation", "counter"),
|
||||
("validation", "cluster"),
|
||||
("validation", "finalize"),
|
||||
]
|
||||
|
||||
|
||||
def test_titanet_batches_keep_variable_length_row_order():
|
||||
features = [
|
||||
np.full((3, 80), 10.0, dtype=np.float32),
|
||||
np.full((5, 80), 20.0, dtype=np.float32),
|
||||
np.full((2, 80), 30.0, dtype=np.float32),
|
||||
np.full((4, 80), 40.0, dtype=np.float32),
|
||||
np.full((6, 80), 50.0, dtype=np.float32),
|
||||
]
|
||||
|
||||
class Session:
|
||||
def __init__(self):
|
||||
self.length_batches = []
|
||||
|
||||
def run(self, output_names, inputs):
|
||||
assert output_names == ["embs"]
|
||||
values = inputs["audio_signal"]
|
||||
lengths = inputs["length"]
|
||||
self.length_batches.append(lengths.tolist())
|
||||
assert values.shape == (len(lengths), 80, max(lengths))
|
||||
return [
|
||||
np.asarray(
|
||||
[[values[index, 0, 0], length] for index, length in enumerate(lengths)],
|
||||
dtype=np.float32,
|
||||
)
|
||||
]
|
||||
|
||||
session = Session()
|
||||
batch4 = runner._infer_titanet_batches(session, features, batch_size=4)
|
||||
batch1 = runner._infer_titanet_batches(Session(), features, batch_size=1)
|
||||
|
||||
assert session.length_batches == [[3, 5, 2, 4], [6]]
|
||||
assert batch4.tolist() == batch1.tolist() == [
|
||||
[10.0, 3.0],
|
||||
[20.0, 5.0],
|
||||
[30.0, 2.0],
|
||||
[40.0, 4.0],
|
||||
[50.0, 6.0],
|
||||
]
|
||||
|
||||
|
||||
def test_titanet_runner_compares_batch_1_and_4_before_finalize(monkeypatch):
|
||||
embeddings = np.asarray(
|
||||
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]], dtype=np.float32
|
||||
)
|
||||
prepared = SimpleNamespace(
|
||||
is_terminal=False,
|
||||
embeddings=embeddings.copy(),
|
||||
row_mapping=((0, 0, [(0, 4)]), (0, 1, [(4, 8)]), (1, 0, [(8, 12)])),
|
||||
counters={"embedding_jobs": 3, "embedding_run_calls": 3, "embedding_rows_dropped": 0},
|
||||
)
|
||||
|
||||
events = []
|
||||
active_interval = {"name": None}
|
||||
|
||||
class Clustering:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
def __call__(self, values):
|
||||
events.append((active_interval["name"], "cluster"))
|
||||
return np.asarray([0, 0, 1], dtype=np.int32)
|
||||
|
||||
fake_sherpa = SimpleNamespace(
|
||||
FastClusteringConfig=lambda **values: SimpleNamespace(**values),
|
||||
FastClustering=Clustering,
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "sherpa_onnx", fake_sherpa)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"estimate_nme",
|
||||
lambda values: (
|
||||
events.append((active_interval["name"], "counter"))
|
||||
or {"status": "complete", "num_clusters": 2, "p": 2}
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"_prepare_titanet_features",
|
||||
lambda mapping, samples: [
|
||||
np.ones((3, 80), dtype=np.float32),
|
||||
np.ones((4, 80), dtype=np.float32),
|
||||
np.ones((5, 80), dtype=np.float32),
|
||||
],
|
||||
)
|
||||
seen_batches = []
|
||||
|
||||
def infer(session, features, batch_size):
|
||||
seen_batches.append((active_interval["name"], batch_size))
|
||||
return embeddings.copy()
|
||||
|
||||
monkeypatch.setattr(runner, "_infer_titanet_batches", infer)
|
||||
measurement_names = iter(
|
||||
("published", "published-embedding", "validation", "validation-embedding")
|
||||
)
|
||||
|
||||
def measure(operation, process, interval_seconds):
|
||||
name = next(measurement_names)
|
||||
previous = active_interval["name"]
|
||||
active_interval["name"] = name
|
||||
try:
|
||||
result = operation()
|
||||
finally:
|
||||
active_interval["name"] = previous
|
||||
return result, {
|
||||
"wall_seconds": 1.0,
|
||||
"user_cpu_seconds": 0.7,
|
||||
"system_cpu_seconds": 0.1,
|
||||
"total_cpu_seconds": 0.8,
|
||||
"average_cpu_cores": 0.8,
|
||||
"peak_rss_bytes": 100,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(runner, "measure_call", measure)
|
||||
|
||||
@contextmanager
|
||||
def native_log():
|
||||
yield io.BytesIO()
|
||||
|
||||
monkeypatch.setattr(runner, "capture_native_stderr", native_log)
|
||||
monkeypatch.setattr(
|
||||
runner,
|
||||
"parse_prepare_timings",
|
||||
lambda log: {"segmentation_seconds": 0.2, "embedding_seconds": 0.4},
|
||||
)
|
||||
|
||||
class Diarization:
|
||||
def __init__(self, labels):
|
||||
self.labels = labels
|
||||
|
||||
def sort_by_start_time(self):
|
||||
return [
|
||||
SimpleNamespace(speaker=int(label), start=index, end=index + 1)
|
||||
for index, label in enumerate(self.labels)
|
||||
]
|
||||
|
||||
class Engine:
|
||||
def prepare(self, samples, workers):
|
||||
assert workers == 1
|
||||
events.append((active_interval["name"], "prepare"))
|
||||
return prepared
|
||||
|
||||
def finalize(self, prepared_value, labels):
|
||||
assert prepared_value is prepared
|
||||
events.append((active_interval["name"], "finalize"))
|
||||
return Diarization(labels)
|
||||
|
||||
cell = _special_cell("titanet-batch", batch_size=4)
|
||||
result, metrics, _, data = runner._run_titanet_batch_recording(
|
||||
Engine(),
|
||||
object(),
|
||||
np.arange(20, dtype=np.float32),
|
||||
cell,
|
||||
{"expected_speakers": 2},
|
||||
0.01,
|
||||
)
|
||||
|
||||
assert seen_batches == [("published-embedding", 4), ("validation-embedding", 1)]
|
||||
assert data["validation_passed"] is True
|
||||
assert data["inference"]["n_equal"] is True
|
||||
assert data["inference"]["labels_equal"] is True
|
||||
assert data["inference"]["final_equal"] is True
|
||||
assert data["counters"]["embedding_run_calls"] == 1
|
||||
assert metrics["measurement_kind"] == "composed-estimate"
|
||||
assert metrics["wall_seconds_is_composed"] is True
|
||||
assert metrics["instrumented_pipeline_wall_seconds"] == 1.0
|
||||
assert metrics["embedding_peak_rss_bytes"] == 100
|
||||
assert metrics["validation_wall_seconds"] == 1.0
|
||||
assert metrics["validation_peak_rss_bytes"] == 100
|
||||
assert events == [
|
||||
("published", "prepare"),
|
||||
("published", "counter"),
|
||||
("published", "cluster"),
|
||||
("published", "finalize"),
|
||||
("validation", "counter"),
|
||||
("validation", "cluster"),
|
||||
("validation", "counter"),
|
||||
("validation", "cluster"),
|
||||
("validation", "finalize"),
|
||||
]
|
||||
assert [item.speaker for item in result.sort_by_start_time()] == [0, 0, 1]
|
||||
Reference in new issue
Block a user