From bd0d7e67368388f39a6f5d6077b2e67431702654 Mon Sep 17 00:00:00 2001 From: Dmitry Dementiev Date: Mon, 31 Aug 2026 15:12:33 +0300 Subject: [PATCH 1/4] =?UTF-8?q?feat(diarization):=20=D0=BF=D0=BE=D0=B4?= =?UTF-8?q?=D0=B3=D0=BE=D1=82=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=20=D1=81=D1=82?= =?UTF-8?q?=D0=B5=D0=BD=D0=B4=20Intel-=D1=8D=D0=BA=D1=81=D0=BF=D0=B5=D1=80?= =?UTF-8?q?=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Зачем: - нужен воспроизводимый Intel-этап перед проверкой кандидатов на Ryzen - sherpa-onnx 1.13.6 открывает настройку шага окна Что: - добавлены строгий manifest v3, планировщик A/B и проверяемая приватная капсула - реализованы экспериментальные NME, модели эмбеддингов и режимы worker/session - версия sherpa-onnx поднята до 1.13.6, зависимости и тесты закреплены Проверка: - uv lock --check - uv run pytest -q - uv run ruff check для изменённых Python-файлов - uv build с проверкой состава wheel --- pyproject.toml | 13 +- scripts/benchmarks/diarization_calibration.py | 30 +- src/local_transcriber/benchmark_experiment.py | 1772 +++++++++++++++++ .../experimental_diarization.py | 1489 ++++++++++++++ tests/test_benchmark_experiment.py | 923 +++++++++ tests/test_diarization_calibration.py | 12 +- tests/test_experimental_diarization.py | 677 +++++++ uv.lock | 201 +- 8 files changed, 5055 insertions(+), 62 deletions(-) create mode 100644 src/local_transcriber/benchmark_experiment.py create mode 100644 src/local_transcriber/experimental_diarization.py create mode 100644 tests/test_benchmark_experiment.py create mode 100644 tests/test_experimental_diarization.py diff --git a/pyproject.toml b/pyproject.toml index e0c55d8..55ae5d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "httpx>=0.28,<1", "socksio>=1.0.0,<2", # При обновлении сверить блок tool.uv.dependency-metadata ниже. - "sherpa-onnx==1.13.5", + "sherpa-onnx==1.13.6", "nvidia-cublas-cu12>=12.4,<13; sys_platform == 'linux' and platform_machine == 'x86_64'", "openvino-genai>=2026.3.0.0,<2026.4; sys_platform != 'darwin' and (platform_machine == 'x86_64' or platform_machine == 'AMD64')", "onnx-asr[cpu,hub]>=0.12,<0.13", @@ -23,6 +23,9 @@ transcribe = "local_transcriber.cli:app" [dependency-groups] dev = [ + "kaldi-native-fbank>=1.22,<2", + "numpy==2.4.3", + "onnx>=1.18,<2", "psutil>=7,<8", "pytest", ] @@ -36,10 +39,14 @@ testpaths = ["tests"] [tool.hatch.build.targets.wheel] packages = ["src/local_transcriber"] +exclude = [ + "src/local_transcriber/benchmark_experiment.py", + "src/local_transcriber/experimental_diarization.py", +] [[tool.uv.dependency-metadata]] # Метаданные wheel-файлов sherpa расходятся, поэтому зависимость от core задана для uv явно. # Версии sherpa, override и core меняются вместе; проверка: uv tree --locked --package sherpa-onnx name = "sherpa-onnx" -version = "1.13.5" -requires-dist = ["sherpa-onnx-core==1.13.5"] +version = "1.13.6" +requires-dist = ["sherpa-onnx-core==1.13.6"] diff --git a/scripts/benchmarks/diarization_calibration.py b/scripts/benchmarks/diarization_calibration.py index 9095f0e..fcf406e 100644 --- a/scripts/benchmarks/diarization_calibration.py +++ b/scripts/benchmarks/diarization_calibration.py @@ -25,7 +25,7 @@ from tempfile import NamedTemporaryFile from typing import Any SCHEMA_VERSION = 2 -EXPECTED_SHERPA_ONNX_VERSION = "1.13.5" +EXPECTED_SHERPA_ONNX_VERSION = "1.13.6" SAMPLE_RATE = 16_000 SAMPLE_WIDTH_BYTES = 2 CHANNELS = 1 @@ -133,7 +133,7 @@ class PeakRssSampler: def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( - "Проверяет и запускает воспроизводимый CPU-benchmark из issue #25. " + "Проверяет и запускает воспроизводимый CPU-benchmark диаризации. " "Приватные пути и результаты остаются вне Git." ), epilog=( @@ -144,7 +144,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--manifest", type=Path, - help="manifest v2; относительные пути считаются от его каталога", + help="manifest v2/v3; относительные пути считаются от его каталога", ) parser.add_argument( "--output", @@ -156,6 +156,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="каталог декодированных WAV и запросов дочерним процессам", ) + parser.add_argument( + "--capsule", + type=Path, + help="исходный ZIP приватной капсулы для проверки handoff", + ) + parser.add_argument( + "--capsule-sha256", + help="ожидаемый внешний SHA-256 ZIP-капсулы", + ) # Оставлено для воспроизводимости старых команд; значение входит в experiment_id. parser.add_argument( "--threads", @@ -201,6 +210,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: missing.append("output") if missing: parser.error("обязательные параметры: " + ", ".join(missing)) + if (args.capsule is None) != (args.capsule_sha256 is None): + parser.error("--capsule и --capsule-sha256 задаются вместе") if args.prepare_asr is not None: asr_missing = [ name @@ -747,7 +758,7 @@ def select_threshold( def parse_stage_timings(stderr: str, sherpa_version: str) -> dict[str, float]: - """Строго разбирает недокументированный debug-контракт sherpa 1.13.5.""" + """Строго разбирает недокументированный debug-контракт sherpa 1.13.6.""" if sherpa_version != EXPECTED_SHERPA_ONNX_VERSION: raise ValueError( f"Неизвестный формат stage timing sherpa-onnx {sherpa_version}" @@ -1536,9 +1547,20 @@ def validation_summary( def main(argv: list[str] | None = None) -> None: args = parse_args(argv) if args.worker is not None: + request = json.loads(args.worker.read_text(encoding="utf-8")) + if request.get("worker_schema_version") == 3: + from local_transcriber.benchmark_experiment import worker_main + + worker_main(args.worker) + return _worker_main(args.worker) return raw_manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + if raw_manifest.get("schema_version") == 3: + from local_transcriber.benchmark_experiment import run_cli + + run_cli(args, raw_manifest) + return validate_manifest(raw_manifest, verify_files=False) manifest = resolve_manifest_paths(raw_manifest, args.manifest.parent) if args.threads is not None: diff --git a/src/local_transcriber/benchmark_experiment.py b/src/local_transcriber/benchmark_experiment.py new file mode 100644 index 0000000..1e5008e --- /dev/null +++ b/src/local_transcriber/benchmark_experiment.py @@ -0,0 +1,1772 @@ +"""Стенд поэтапного CPU-эксперимента диаризации.""" + +from __future__ import annotations + +import copy +import hashlib +import importlib.metadata +import json +import math +import os +import platform +import subprocess +import sys +import tempfile +import zipfile +from collections.abc import Callable, Iterable, Mapping, Sequence +from itertools import pairwise +from pathlib import Path, PurePosixPath +from typing import Any + +import numpy as np + +SCHEMA_VERSION = 3 +EXPECTED_DEPENDENCIES = { + "sherpa-onnx": "1.13.6", + "sherpa-onnx-core": "1.13.6", +} +RUNTIME_PACKAGES = ( + "sherpa-onnx", + "sherpa-onnx-core", + "onnxruntime", + "numpy", + "onnx", + "kaldi-native-fbank", + "psutil", +) +EXPERIMENT_BUILD_SHA256_ENV = "LOCAL_TRANSCRIBER_EXPERIMENT_BUILD_SHA256" +SHA256_LENGTH = 64 +WINDOW_SHIFTS = (0.1, 0.15, 0.2, 0.25, 0.5) +CELL_STAGES = {"intel", "ryzen"} +CELL_PHASES = {"baseline", "mechanism", "window", "model", "execution", "combination"} +COUNTERS = {"threshold", "eigengap", "nme", "known"} +INFERENCE_MODES = { + "sequential", + "shared-session", + "separate-session", + "titanet-batch", +} + +TOP_LEVEL_KEYS = { + "schema_version", + "experiment", + "machine", + "artifacts", + "recordings", + "calibration", + "qdq", + "cells", + "handoff", + "rss_sample_interval_ms", +} +EXPERIMENT_KEYS = { + "id", + "stage", + "source_commit", + "handoff_commit", + "sherpa_patch_sha256", + "python_version", + "dependencies", +} +MACHINE_KEYS = { + "id", + "sku", + "physical_cores", + "logical_cores", + "ram_bytes", + "os", + "cpu_flags", + "power", +} +POWER_KEYS = { + "supply", + "profile", + "governor", + "turbo", + "power_limits", + "temperature_celsius", + "throttling", + "swap_total_bytes", + "swap_used_bytes", +} +ARTIFACT_KEYS = { + "id", + "kind", + "path", + "sha256", + "size_bytes", + "source_url", + "revision", + "license", +} +RECORDING_KEYS = { + "id", + "role", + "path", + "sha256", + "start", + "duration", + "expected_speakers", + "reference", + "asr_words", +} +SIDE_FILE_KEYS = {"path", "sha256", "format"} +CALIBRATION_KEYS = { + "id", + "path", + "sha256", + "source_id", + "source_sha256", + "source_start", + "source_duration", + "derived", +} +QDQ_KEYS = { + "enabled", + "source_ids", + "preprocessing", + "reader", + "quant_pre_process", + "quantize_static", + "excluded_nodes", + "tool_versions", + "graph_artifact_id", + "skip_reason", +} +CELL_KEYS = { + "name", + "stage", + "phase", + "recording_ids", + "window_shift_ratio", + "segmentation_artifact_id", + "embedding_artifact_id", + "counter", + "clustering", + "inference", + "repetition", + "schedule_position", + "pair_id", + "pair_role", + "enabled", + "skip_reason", + "build_sha256", +} +CLUSTERING_KEYS = {"mode", "threshold", "num_clusters"} +INFERENCE_KEYS = { + "mode", + "outer_workers", + "session_count", + "intra_op_threads", + "inter_op_threads", + "batch_size", +} +HANDOFF_KEYS = { + "outcome", + "source_commit", + "candidate_recipes", + "public_report_sha256", + "capsule_id", + "capsule_sha256", +} +CANDIDATE_EVIDENCE_KEYS = { + "name", + "cell_id", + "result_file", + "result_sha256", + "mandatory_passed", + "memory_passed", + "diagnostic_approved", +} +RUNTIME_GUARD_KEYS = { + "supply", + "profile", + "governor", + "turbo", + "power_limits", + "temperature_celsius", + "throttling", + "throttle_count", + "swap_used_bytes", + "swap_sin_bytes", + "swap_sout_bytes", +} + + +def _exact_keys(value: Mapping[str, Any], expected: set[str], label: str) -> None: + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise ValueError( + f"{label}: неверный набор полей; нет={missing}, лишние={extra}" + ) + + +def _require_id(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or any(ch.isspace() for ch in value): + raise ValueError(f"{label}: нужен непустой идентификатор без пробелов") + return value + + +def _require_sha256(value: Any, label: str, *, optional: bool = False) -> str | None: + if optional and value is None: + return None + if ( + not isinstance(value, str) + or len(value) != SHA256_LENGTH + or any(ch not in "0123456789abcdef" for ch in value) + ): + raise ValueError(f"{label}: нужен SHA-256 в нижнем регистре") + return value + + +def _require_commit(value: Any, label: str, *, optional: bool = False) -> str | None: + if optional and value is None: + return None + if ( + not isinstance(value, str) + or len(value) not in {40, 64} + or any(ch not in "0123456789abcdef" for ch in value) + ): + raise ValueError(f"{label}: нужен полный Git object ID в нижнем регистре") + return value + + +def _require_version(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise ValueError(f"{label}: нужна точная непустая версия") + return value + + +def _require_relative_path(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{label}: нужен непустой относительный путь") + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or "\\" in value: + raise ValueError(f"{label}: путь должен быть относительным и безопасным") + return value + + +def _require_positive(value: Any, label: str, *, allow_zero: bool = False) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{label}: нужно число") + result = float(value) + if not math.isfinite(result) or result < 0 or (not allow_zero and result == 0): + raise ValueError(f"{label}: нужно положительное конечное число") + return result + + +def _require_integer(value: Any, label: str, *, allow_zero: bool = False) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{label}: нужно целое число") + if value < 0 or (not allow_zero and value == 0): + raise ValueError(f"{label}: нужно положительное целое число") + return value + + +def _unique_ids(items: Sequence[Mapping[str, Any]], label: str) -> set[str]: + identifiers = [_require_id(item.get("id"), f"{label}.id") for item in items] + if len(identifiers) != len(set(identifiers)): + raise ValueError(f"{label}: идентификаторы должны быть уникальны") + return set(identifiers) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_file(item: Mapping[str, Any], label: str) -> None: + path = Path(item["path"]) + if not path.is_file(): + raise ValueError(f"{label}: файл недоступен") + if path.stat().st_size != item.get("size_bytes", path.stat().st_size): + raise ValueError(f"{label}: размер файла не совпал") + if file_sha256(path) != item["sha256"]: + raise ValueError(f"{label}: SHA-256 не совпал") + + +def resolve_manifest_paths( + manifest: Mapping[str, Any], manifest_dir: Path +) -> dict[str, Any]: + """Разрешает приватные пути, не добавляя их в семантическую идентичность.""" + result = copy.deepcopy(manifest) + base = manifest_dir.resolve() + + def resolve(value: str) -> str: + path = Path(value) + if path.is_absolute(): + return str(path.resolve()) + resolved = (base / path).resolve() + if not resolved.is_relative_to(base): + raise ValueError("Относительный путь выходит за корень manifest") + return str(resolved) + + for artifact in result["artifacts"]: + artifact["path"] = resolve(artifact["path"]) + for recording in result["recordings"]: + recording["path"] = resolve(recording["path"]) + for key in ("reference", "asr_words"): + if recording[key] is not None: + recording[key]["path"] = resolve(recording[key]["path"]) + for source in result["calibration"]: + source["path"] = resolve(source["path"]) + return result + + +def validate_manifest( + manifest: Mapping[str, Any], + *, + verify_files: bool = True, + require_relative_paths: bool = False, +) -> list[str]: + """Проверяет manifest v3 и возвращает безопасные предупреждения.""" + _exact_keys(manifest, TOP_LEVEL_KEYS, "manifest") + if manifest["schema_version"] != SCHEMA_VERSION: + raise ValueError("Поддерживается manifest schema_version=3") + + experiment = manifest["experiment"] + _exact_keys(experiment, EXPERIMENT_KEYS, "experiment") + _require_id(experiment["id"], "experiment.id") + if experiment["stage"] not in CELL_STAGES: + raise ValueError("experiment.stage: допустимы intel и ryzen") + _require_commit(experiment["source_commit"], "experiment.source_commit") + _require_commit( + experiment["handoff_commit"], "experiment.handoff_commit", optional=True + ) + _require_sha256( + experiment["sherpa_patch_sha256"], + "experiment.sherpa_patch_sha256", + optional=True, + ) + _require_version(experiment["python_version"], "experiment.python_version") + if experiment["stage"] == "intel" and experiment["handoff_commit"] is not None: + raise ValueError("Intel-этап не принимает handoff_commit") + if experiment["stage"] == "ryzen" and experiment["handoff_commit"] is None: + raise ValueError("Ryzen-этап требует handoff_commit") + dependencies = experiment["dependencies"] + _exact_keys(dependencies, set(RUNTIME_PACKAGES), "experiment.dependencies") + for name, version in dependencies.items(): + _require_version(version, f"experiment.dependencies.{name}") + if any(dependencies[name] != version for name, version in EXPECTED_DEPENDENCIES.items()): + raise ValueError("Версии sherpa-onnx и sherpa-onnx-core должны быть 1.13.6") + + machine = manifest["machine"] + _exact_keys(machine, MACHINE_KEYS, "machine") + _require_id(machine["id"], "machine.id") + if not isinstance(machine["sku"], str) or not machine["sku"].strip(): + raise ValueError("machine.sku: нужна непустая строка") + physical = _require_integer(machine["physical_cores"], "physical_cores") + logical = _require_integer(machine["logical_cores"], "logical_cores") + if logical < physical: + raise ValueError("logical_cores не может быть меньше physical_cores") + _require_positive(machine["ram_bytes"], "machine.ram_bytes") + if not isinstance(machine["cpu_flags"], list) or not machine["cpu_flags"]: + raise ValueError("machine.cpu_flags: нужен непустой список") + if machine["cpu_flags"] != sorted(set(machine["cpu_flags"])): + raise ValueError("machine.cpu_flags: список должен быть сортирован без дублей") + _exact_keys(machine["power"], POWER_KEYS, "machine.power") + _require_positive( + machine["power"]["temperature_celsius"], + "machine.power.temperature_celsius", + allow_zero=True, + ) + _require_positive( + machine["power"]["swap_total_bytes"], + "machine.power.swap_total_bytes", + allow_zero=True, + ) + _require_positive( + machine["power"]["swap_used_bytes"], + "machine.power.swap_used_bytes", + allow_zero=True, + ) + if not isinstance(machine["power"]["throttling"], bool): + raise TypeError("machine.power.throttling: нужен bool") + if machine["power"]["supply"] != "ac": + raise ValueError("machine.power.supply: эксперимент выполняется от сети") + if machine["power"]["profile"] != "balanced": + raise ValueError("machine.power.profile: нужен стабильный balanced") + if machine["power"]["throttling"]: + raise ValueError("machine.power.throttling: исходное состояние нестабильно") + if not isinstance(machine["power"]["turbo"], bool): + raise TypeError("machine.power.turbo: нужен bool") + + artifact_ids = _unique_ids(manifest["artifacts"], "artifacts") + for artifact in manifest["artifacts"]: + _exact_keys(artifact, ARTIFACT_KEYS, f"artifact:{artifact['id']}") + _require_sha256(artifact["sha256"], f"artifact:{artifact['id']}.sha256") + _require_positive(artifact["size_bytes"], f"artifact:{artifact['id']}.size") + if require_relative_paths: + _require_relative_path(artifact["path"], f"artifact:{artifact['id']}.path") + if verify_files: + _verify_file(artifact, f"artifact:{artifact['id']}") + + recording_ids = _unique_ids(manifest["recordings"], "recordings") + recording_sets_by_role: dict[str, set[str]] = {"control": set(), "full": set()} + control_hashes: set[str] = set() + for recording in manifest["recordings"]: + _exact_keys(recording, RECORDING_KEYS, f"recording:{recording['id']}") + if recording["role"] not in {"control", "full"}: + raise ValueError(f"recording:{recording['id']}: неизвестная роль") + sha256 = _require_sha256( + recording["sha256"], f"recording:{recording['id']}.sha256" + ) + if recording["role"] == "control": + control_hashes.add(sha256) + recording_sets_by_role[recording["role"]].add(recording["id"]) + _require_positive( + recording["duration"], f"recording:{recording['id']}.duration" + ) + _require_positive( + recording["start"], f"recording:{recording['id']}.start", allow_zero=True + ) + _require_integer( + recording["expected_speakers"], + f"recording:{recording['id']}.expected_speakers", + ) + for key in ("reference", "asr_words"): + side = recording[key] + if side is None: + if key == "asr_words": + raise ValueError(f"recording:{recording['id']}: нужен asr_words") + continue + _exact_keys(side, SIDE_FILE_KEYS, f"recording:{recording['id']}.{key}") + expected_format = ( + "hypescribe_markdown" + if key == "reference" + else "local_transcriber_words_v1" + ) + if side["format"] != expected_format: + raise ValueError( + f"recording:{recording['id']}.{key}: неверный формат" + ) + if require_relative_paths: + _require_relative_path( + side["path"], f"recording:{recording['id']}.{key}.path" + ) + _require_sha256(side["sha256"], f"recording:{recording['id']}.{key}") + if verify_files: + _verify_file(side, f"recording:{recording['id']}.{key}") + if require_relative_paths: + _require_relative_path( + recording["path"], f"recording:{recording['id']}.path" + ) + if verify_files: + _verify_file(recording, f"recording:{recording['id']}") + + calibration_ids = _unique_ids(manifest["calibration"], "calibration") + original_calibration_hashes = { + source["sha256"] + for source in manifest["calibration"] + if source.get("derived") is False + } + source_hashes_by_id: dict[str, str] = {} + missing_calibration: list[str] = [] + for source in manifest["calibration"]: + _exact_keys(source, CALIBRATION_KEYS, f"calibration:{source['id']}") + _require_sha256(source["sha256"], f"calibration:{source['id']}.sha256") + if require_relative_paths: + _require_relative_path(source["path"], f"calibration:{source['id']}.path") + source_sha = _require_sha256( + source["source_sha256"], f"calibration:{source['id']}.source_sha256" + ) + if not isinstance(source["derived"], bool): + raise TypeError(f"calibration:{source['id']}.derived: нужен bool") + if source_sha in control_hashes: + raise ValueError( + f"calibration:{source['id']}: контрольная запись запрещена" + ) + if not source["derived"] and source["sha256"] != source_sha: + raise ValueError( + f"calibration:{source['id']}: исходник должен совпадать с source_sha256" + ) + if source["derived"] and source_sha not in original_calibration_hashes: + raise ValueError( + f"calibration:{source['id']}: исходный материал не включен в manifest" + ) + known_source_hash = source_hashes_by_id.setdefault(source["source_id"], source_sha) + if known_source_hash != source_sha: + raise ValueError( + f"calibration:{source['id']}: source_id ссылается на разные исходники" + ) + _require_positive( + source["source_start"], + f"calibration:{source['id']}.source_start", + allow_zero=True, + ) + _require_positive( + source["source_duration"], f"calibration:{source['id']}.source_duration" + ) + if verify_files: + path = Path(source["path"]) + if not path.is_file() or file_sha256(path) != source["sha256"]: + missing_calibration.append(source["id"]) + + qdq = manifest["qdq"] + _exact_keys(qdq, QDQ_KEYS, "qdq") + if not isinstance(qdq["enabled"], bool): + raise TypeError("qdq.enabled: нужен bool") + if not isinstance(qdq["source_ids"], list) or len(qdq["source_ids"]) != len( + set(qdq["source_ids"]) + ): + raise ValueError("qdq.source_ids: нужен список без дублей") + if not set(qdq["source_ids"]) <= calibration_ids: + raise ValueError("qdq.source_ids: неизвестный калибровочный источник") + if qdq["enabled"]: + if not qdq["source_ids"]: + missing_calibration.append("provenance") + if qdq["graph_artifact_id"] is None: + raise ValueError("qdq.graph_artifact_id: для enabled QDQ нужен граф") + if qdq["skip_reason"] is not None: + raise ValueError("qdq.skip_reason: для enabled QDQ должен быть null") + elif not isinstance(qdq["skip_reason"], str) or not qdq["skip_reason"].strip(): + raise ValueError("qdq.skip_reason: для disabled QDQ нужна причина") + if ( + qdq["graph_artifact_id"] is not None + and qdq["graph_artifact_id"] not in artifact_ids + ): + raise ValueError("qdq.graph_artifact_id: неизвестный артефакт") + + cell_ids: set[str] = set() + schedule_positions: set[int] = set() + for index, cell in enumerate(manifest["cells"]): + _exact_keys(cell, CELL_KEYS, f"cell:{index}") + _require_id(cell["name"], f"cell:{index}.name") + if cell["stage"] != experiment["stage"]: + raise ValueError(f"cell:{cell['name']}: нарушена граница Intel/Ryzen") + if cell["phase"] not in CELL_PHASES: + raise ValueError(f"cell:{cell['name']}: неизвестная фаза") + if ( + not cell["recording_ids"] + or len(cell["recording_ids"]) != len(set(cell["recording_ids"])) + or not set(cell["recording_ids"]) <= recording_ids + ): + raise ValueError(f"cell:{cell['name']}: неизвестная запись") + selected_recordings = set(cell["recording_ids"]) + complete_sets = [items for items in recording_sets_by_role.values() if items] + if selected_recordings not in complete_sets: + raise ValueError( + f"cell:{cell['name']}: нужен полный набор control или full" + ) + ratio = _require_positive( + cell["window_shift_ratio"], f"cell:{cell['name']}.window_shift_ratio" + ) + if ratio not in WINDOW_SHIFTS: + raise ValueError(f"cell:{cell['name']}: шаг окна не входит в сетку") + for field in ("segmentation_artifact_id", "embedding_artifact_id"): + if cell[field] not in artifact_ids: + raise ValueError(f"cell:{cell['name']}.{field}: неизвестный артефакт") + if cell["counter"] not in COUNTERS: + raise ValueError(f"cell:{cell['name']}: неизвестный счетчик") + _exact_keys( + cell["clustering"], CLUSTERING_KEYS, f"cell:{cell['name']}.clustering" + ) + clustering = cell["clustering"] + expected_clustering_mode = ( + "threshold" if cell["counter"] == "threshold" else "counter" + ) + if clustering["mode"] != expected_clustering_mode: + raise ValueError(f"cell:{cell['name']}: counter и clustering.mode не совпали") + _require_positive(clustering["threshold"], f"cell:{cell['name']}.threshold") + if clustering["num_clusters"] is not None: + _require_integer( + clustering["num_clusters"], f"cell:{cell['name']}.num_clusters" + ) + _exact_keys(cell["inference"], INFERENCE_KEYS, f"cell:{cell['name']}.inference") + inference = cell["inference"] + if inference["mode"] not in INFERENCE_MODES: + raise ValueError(f"cell:{cell['name']}: неизвестный inference mode") + outer = _require_integer(inference["outer_workers"], "outer_workers") + sessions = _require_integer(inference["session_count"], "session_count") + intra = _require_integer(inference["intra_op_threads"], "intra_op_threads") + inter = _require_integer(inference["inter_op_threads"], "inter_op_threads") + batch = _require_integer(inference["batch_size"], "batch_size") + if outer > physical or max(outer, sessions * intra) > physical: + raise ValueError(f"cell:{cell['name']}: превышен бюджет физических ядер") + if inter != 1: + raise ValueError(f"cell:{cell['name']}: inter_op_threads должен быть 1") + if inference["mode"] != "titanet-batch" and batch != 1: + raise ValueError(f"cell:{cell['name']}: batch применим только к TitaNet") + if inference["mode"] == "shared-session" and sessions != 1: + raise ValueError( + f"cell:{cell['name']}: shared-session требует одну session" + ) + if inference["mode"] == "shared-session" and outer == 1: + raise ValueError( + f"cell:{cell['name']}: shared-session требует несколько worker" + ) + if inference["mode"] == "separate-session" and ( + outer != physical or sessions != physical or intra != 1 + ): + raise ValueError( + f"cell:{cell['name']}: separate-session требует P worker/session и intra-op=1" + ) + if inference["mode"] == "titanet-batch" and ( + outer != 1 + or sessions != 1 + or intra != physical + or batch not in {1, 4} + or cell["counter"] != "nme" + ): + raise ValueError( + f"cell:{cell['name']}: нарушен контракт TitaNet batch" + ) + repetition = _require_integer(cell["repetition"], "repetition") + position = _require_integer(cell["schedule_position"], "schedule_position") + if position in schedule_positions: + raise ValueError("schedule_position должен быть уникален") + schedule_positions.add(position) + if cell["pair_role"] not in {None, "baseline", "candidate"}: + raise ValueError(f"cell:{cell['name']}: неверная роль A/B") + if (cell["pair_id"] is None) != (cell["pair_role"] is None): + raise ValueError( + f"cell:{cell['name']}: pair_id и pair_role задаются вместе" + ) + if not isinstance(cell["enabled"], bool): + raise TypeError(f"cell:{cell['name']}: enabled должен быть bool") + _require_sha256(cell["build_sha256"], f"cell:{cell['name']}.build_sha256") + identity = make_cell_id(manifest, cell) + if identity in cell_ids: + raise ValueError(f"cell:{cell['name']}: дублирующая семантическая ячейка") + cell_ids.add(identity) + del repetition + + _validate_ab_schedule(manifest["cells"]) + combination_cells = { + cell["name"]: cell + for cell in manifest["cells"] + if cell["phase"] == "combination" and cell["enabled"] + } + model_cells = [cell for cell in manifest["cells"] if cell["phase"] == "model"] + for cell in manifest["cells"]: + if cell["inference"]["mode"] != "titanet-batch": + continue + if not any( + model["embedding_artifact_id"] == cell["embedding_artifact_id"] + and model["counter"] == "nme" + and model["schedule_position"] < cell["schedule_position"] + and model["enabled"] + for model in model_cells + ): + raise ValueError( + f"cell:{cell['name']}: TitaNet batch требует предшествующую model-ячейку NME" + ) + _validate_handoff(manifest["handoff"], experiment, combination_cells, manifest) + _require_positive(manifest["rss_sample_interval_ms"], "rss_sample_interval_ms") + + warnings = [] + if missing_calibration: + warnings.append( + "QDQ пропущен: калибровочный материал или происхождение не подтверждены" + ) + return warnings + + +def _validate_ab_schedule(cells: Sequence[Mapping[str, Any]]) -> None: + pairs: dict[str, list[Mapping[str, Any]]] = {} + for cell in cells: + if cell["pair_id"] is not None: + pairs.setdefault(cell["pair_id"], []).append(cell) + first_roles = [] + for pair_id, pair_cells in pairs.items(): + ordered = sorted(pair_cells, key=lambda item: item["schedule_position"]) + roles = [item["pair_role"] for item in ordered] + if len(ordered) != 2 or set(roles) != {"baseline", "candidate"}: + raise ValueError(f"A/B {pair_id}: нужна ровно одна пара baseline/candidate") + positions = [item["schedule_position"] for item in ordered] + if positions[1] != positions[0] + 1: + raise ValueError(f"A/B {pair_id}: позиции пары должны быть соседними") + first_roles.append((ordered[0]["schedule_position"], roles[0])) + first_roles.sort() + if any(left[1] == right[1] for left, right in pairwise(first_roles)): + raise ValueError("A/B: начальная роль должна меняться между парами") + + +def _validate_handoff( + handoff: Mapping[str, Any], + experiment: Mapping[str, Any], + combination_cells: Mapping[str, Mapping[str, Any]], + manifest: Mapping[str, Any], +) -> None: + _exact_keys(handoff, HANDOFF_KEYS, "handoff") + if handoff["outcome"] not in {"pending", "handoff", "stop-before-ryzen"}: + raise ValueError("handoff.outcome: неизвестный итог") + candidates = handoff["candidate_recipes"] + if not isinstance(candidates, list) or len(candidates) > 3: + raise ValueError("handoff: допускается не больше трех кандидатов") + if handoff["outcome"] == "stop-before-ryzen" and candidates: + raise ValueError("stop-before-ryzen требует ноль кандидатов") + if handoff["outcome"] == "handoff" and not candidates: + raise ValueError("handoff требует от одного до трех кандидатов") + candidate_names = [] + for index, candidate in enumerate(candidates): + if not isinstance(candidate, Mapping): + raise TypeError(f"handoff.candidate_recipes:{index}: нужен объект evidence") + _exact_keys( + candidate, + CANDIDATE_EVIDENCE_KEYS, + f"handoff.candidate_recipes:{index}", + ) + name = _require_id(candidate["name"], f"handoff.candidate_recipes:{index}.name") + candidate_names.append(name) + cell = combination_cells.get(name) + if cell is None: + raise ValueError( + "handoff: кандидат не описан включенной combination-ячейкой" + ) + expected_cell_id = make_cell_id(manifest, cell) + if candidate["cell_id"] != expected_cell_id: + raise ValueError(f"handoff:{name}: cell_id не совпадает с manifest") + _require_sha256(candidate["result_sha256"], f"handoff:{name}.result_sha256") + _require_relative_path(candidate["result_file"], f"handoff:{name}.result_file") + for gate in ("mandatory_passed", "memory_passed", "diagnostic_approved"): + if candidate[gate] is not True: + raise ValueError(f"handoff:{name}: не пройден gate {gate}") + if len(candidate_names) != len(set(candidate_names)): + raise ValueError("handoff: рецепты кандидатов должны быть уникальны") + _require_commit(handoff["source_commit"], "handoff.source_commit", optional=True) + _require_sha256( + handoff["public_report_sha256"], "handoff.public_report_sha256", optional=True + ) + _require_sha256(handoff["capsule_sha256"], "handoff.capsule_sha256", optional=True) + if experiment["stage"] == "ryzen" and handoff["outcome"] == "pending": + raise ValueError("Ryzen-этап не принимает pending handoff") + finalized = handoff["outcome"] != "pending" + required = ( + handoff["source_commit"], + handoff["public_report_sha256"], + handoff["capsule_id"], + ) + if finalized and any(value is None for value in required): + raise ValueError("Итог handoff требует commit, отчет и приватную капсулу") + final_links = (*required, handoff["capsule_sha256"]) + if not finalized and any(value is not None for value in final_links): + raise ValueError("Pending handoff не должен содержать итоговые ссылки") + if finalized and handoff["source_commit"] != experiment["source_commit"]: + raise ValueError("Commit handoff не совпадает с исходным commit эксперимента") + if handoff["capsule_id"] is not None: + _require_id(handoff["capsule_id"], "handoff.capsule_id") + + +def semantic_manifest(manifest: Mapping[str, Any]) -> dict[str, Any]: + """Возвращает идентичность manifest без локальных путей и приватного текста.""" + result = copy.deepcopy(manifest) + result.pop("handoff", None) + for artifact in result["artifacts"]: + artifact.pop("path", None) + for recording in result["recordings"]: + recording.pop("path", None) + for key in ("reference", "asr_words"): + if recording[key] is not None: + recording[key].pop("path", None) + for source in result["calibration"]: + source.pop("path", None) + return result + + +def _canonical_json_bytes(value: Any) -> bytes: + return json.dumps( + value, ensure_ascii=True, sort_keys=True, separators=(",", ":") + ).encode() + + +def _digest_json(value: Any) -> str: + return hashlib.sha256(_canonical_json_bytes(value)).hexdigest() + + +def make_experiment_id(manifest: Mapping[str, Any]) -> str: + return _digest_json(semantic_manifest(manifest)) + + +def make_cell_id(manifest: Mapping[str, Any], cell: Mapping[str, Any]) -> str: + artifact_hashes = { + item["id"]: item["sha256"] + for item in manifest["artifacts"] + if item["id"] + in {cell["segmentation_artifact_id"], cell["embedding_artifact_id"]} + } + recording_inputs = { + item["id"]: { + "media_sha256": item["sha256"], + "start": item["start"], + "duration": item["duration"], + "expected_speakers": item["expected_speakers"], + "reference": ( + None + if item["reference"] is None + else { + "sha256": item["reference"]["sha256"], + "format": item["reference"]["format"], + } + ), + "asr_words": { + "sha256": item["asr_words"]["sha256"], + "format": item["asr_words"]["format"], + }, + } + for item in manifest["recordings"] + if item["id"] in cell["recording_ids"] + } + qdq_identity = None + if cell["embedding_artifact_id"] == manifest["qdq"]["graph_artifact_id"]: + selected_sources = set(manifest["qdq"]["source_ids"]) + qdq_identity = { + "recipe": manifest["qdq"], + "calibration": [ + {key: value for key, value in source.items() if key != "path"} + for source in manifest["calibration"] + if source["id"] in selected_sources + ], + } + identity = { + "experiment": manifest["experiment"], + "machine": manifest["machine"], + "artifact_hashes": artifact_hashes, + "recording_inputs": recording_inputs, + "qdq": qdq_identity, + "cell": cell, + } + return _digest_json(identity) + + +def _linux_cpu_identity() -> tuple[str | None, list[str]]: + cpuinfo = Path("/proc/cpuinfo") + if not cpuinfo.is_file(): + return platform.processor() or None, [] + sku = None + flags: list[str] = [] + for line in cpuinfo.read_text(encoding="utf-8", errors="replace").splitlines(): + key, separator, value = line.partition(":") + if not separator: + continue + if key.strip() == "model name" and sku is None: + sku = value.strip() + elif key.strip() in {"flags", "Features"} and not flags: + flags = sorted(set(value.split())) + if sku is not None and flags: + break + return sku or platform.processor() or None, flags + + +def environment_snapshot() -> dict[str, Any]: + import psutil + + packages = { + name: importlib.metadata.version(name) + for name in RUNTIME_PACKAGES + } + sku, cpu_flags = _linux_cpu_identity() + memory = psutil.virtual_memory() + swap = psutil.swap_memory() + return { + "packages": packages, + "python": platform.python_version(), + "platform": platform.platform(), + "logical_cores": os.cpu_count(), + "physical_cores": psutil.cpu_count(logical=False), + "cpu_sku": sku, + "cpu_flags": cpu_flags, + "ram_bytes": memory.total, + "swap_total_bytes": swap.total, + "swap_used_bytes": swap.used, + "build_sha256": os.environ.get(EXPERIMENT_BUILD_SHA256_ENV), + } + + +def validate_environment( + manifest: Mapping[str, Any], environment: Mapping[str, Any] +) -> None: + experiment = manifest["experiment"] + for name, expected in experiment["dependencies"].items(): + if environment["packages"].get(name) != expected: + raise ValueError(f"Ожидался {name}=={expected}") + if environment.get("python") != experiment["python_version"]: + raise ValueError( + f"Ожидался Python {experiment['python_version']}" + ) + runtime_build = _require_sha256( + environment.get("build_sha256"), + f"переменная {EXPERIMENT_BUILD_SHA256_ENV}", + ) + expected_builds = { + cell["build_sha256"] for cell in manifest["cells"] if cell["enabled"] + } + if expected_builds != {runtime_build}: + raise ValueError("Хеш runtime-сборки не совпал с enabled-ячейками") + machine = manifest["machine"] + if environment.get("platform") != machine["os"]: + raise ValueError("ОС не совпала с manifest") + if environment["logical_cores"] != machine["logical_cores"]: + raise ValueError("Число логических ядер не совпало с manifest") + if environment.get("physical_cores") != machine["physical_cores"]: + raise ValueError("Число физических ядер не совпало с manifest") + if environment.get("cpu_sku") != machine["sku"]: + raise ValueError("CPU SKU не совпал с manifest") + if environment.get("ram_bytes") != machine["ram_bytes"]: + raise ValueError("Объем RAM не совпал с manifest") + if environment.get("swap_total_bytes") != machine["power"]["swap_total_bytes"]: + raise ValueError("Объем swap не совпал с manifest") + if environment.get("cpu_flags") != machine["cpu_flags"]: + raise ValueError("CPUID/ISA не совпали с manifest") + + +def _read_optional_text(path: Path) -> str | None: + try: + return path.read_text(encoding="utf-8").strip() + except OSError: + return None + + +def _runtime_supply() -> str | None: + for device in sorted(Path("/sys/class/power_supply").glob("*")): + kind = _read_optional_text(device / "type") + online = _read_optional_text(device / "online") + if kind in {"Mains", "USB", "USB_PD"} and online == "1": + return "ac" + return None + + +def _runtime_governor() -> str | None: + values = { + value + for path in Path("/sys/devices/system/cpu").glob( + "cpu[0-9]*/cpufreq/scaling_governor" + ) + if (value := _read_optional_text(path)) + } + return next(iter(values)) if len(values) == 1 else None + + +def _runtime_turbo() -> bool | None: + no_turbo = _read_optional_text( + Path("/sys/devices/system/cpu/intel_pstate/no_turbo") + ) + if no_turbo in {"0", "1"}: + return no_turbo == "0" + boost = _read_optional_text(Path("/sys/devices/system/cpu/cpufreq/boost")) + if boost in {"0", "1"}: + return boost == "1" + return None + + +def _runtime_power_limits() -> dict[str, str]: + limits = {} + for path in sorted( + Path("/sys/class/powercap").glob( + "intel-rapl*/constraint_*_power_limit_uw" + ) + ): + value = _read_optional_text(path) + if value is not None: + limits[path.name] = value + return limits + + +def _runtime_temperature() -> float | None: + import psutil + + try: + values = [ + float(item.current) + for entries in psutil.sensors_temperatures().values() + for item in entries + if item.current is not None and math.isfinite(float(item.current)) + ] + except (AttributeError, OSError): + return None + return max(values, default=None) + + +def _runtime_throttle_count() -> int | None: + values = [] + for path in Path("/sys/devices/system/cpu").glob( + "cpu[0-9]*/thermal_throttle/*_throttle_count" + ): + value = _read_optional_text(path) + if value is not None and value.isdigit(): + values.append(int(value)) + return sum(values) if values else None + + +def runtime_guard_snapshot() -> dict[str, Any]: + """Снимает изменяемые условия непосредственно вокруг одной ячейки.""" + override = os.environ.get("LOCAL_TRANSCRIBER_RUNTIME_GUARD_JSON") + if override is not None: + result = json.loads(override) + _exact_keys(result, RUNTIME_GUARD_KEYS, "runtime guard") + return result + + import psutil + + swap = psutil.swap_memory() + return { + "supply": _runtime_supply(), + "profile": _read_optional_text(Path("/sys/firmware/acpi/platform_profile")), + "governor": _runtime_governor(), + "turbo": _runtime_turbo(), + "power_limits": _runtime_power_limits(), + "temperature_celsius": _runtime_temperature(), + "throttling": False, + "throttle_count": _runtime_throttle_count(), + "swap_used_bytes": int(swap.used), + "swap_sin_bytes": int(swap.sin), + "swap_sout_bytes": int(swap.sout), + } + + +def validate_runtime_guard_transition( + manifest: Mapping[str, Any], + before: Mapping[str, Any], + after: Mapping[str, Any], +) -> list[str]: + """Возвращает стабильные коды причин, по которым прогон нельзя учитывать.""" + _exact_keys(before, RUNTIME_GUARD_KEYS, "runtime guard before") + _exact_keys(after, RUNTIME_GUARD_KEYS, "runtime guard after") + expected = manifest["machine"]["power"] + reasons = [] + for key in ("supply", "profile", "governor", "turbo"): + if before[key] != expected[key] or after[key] != expected[key]: + reasons.append(f"{key}-mismatch") + elif before[key] != after[key]: + reasons.append(f"{key}-changed") + if before["power_limits"] != after["power_limits"]: + reasons.append("power-limits-changed") + if before["throttling"] or after["throttling"]: + reasons.append("throttling") + if before["throttle_count"] is None or after["throttle_count"] is None: + reasons.append("throttle-count-unavailable") + elif before["throttle_count"] != after["throttle_count"]: + reasons.append("throttle-count-changed") + for key in ("swap_sin_bytes", "swap_sout_bytes"): + if before[key] != after[key]: + reasons.append(f"{key}-changed") + if after["swap_used_bytes"] > before["swap_used_bytes"]: + reasons.append("swap-used-increased") + return reasons + + +def safe_plan( + manifest: Mapping[str, Any], warnings: Sequence[str], environment: Mapping[str, Any] +) -> dict[str, Any]: + """Возвращает план без локальных путей, текста и интервалов записей.""" + cells = sorted(manifest["cells"], key=lambda item: item["schedule_position"]) + qdq_available = not warnings and manifest["qdq"]["enabled"] + qdq_artifact_id = manifest["qdq"]["graph_artifact_id"] + return { + "status": "valid", + "schema_version": SCHEMA_VERSION, + "experiment_id": make_experiment_id(manifest), + "stage": manifest["experiment"]["stage"], + "machine_id": manifest["machine"]["id"], + "recording_ids": [item["id"] for item in manifest["recordings"]], + "artifact_hashes": { + item["id"]: item["sha256"] for item in manifest["artifacts"] + }, + "cells": [ + { + "name": item["name"], + "cell_id": make_cell_id(manifest, item), + "phase": item["phase"], + "position": item["schedule_position"], + "enabled": item["enabled"] + and (qdq_available or item["embedding_artifact_id"] != qdq_artifact_id), + } + for item in cells + ], + "qdq_available": qdq_available, + "warnings": list(warnings), + "dependencies": environment["packages"], + "python_version": environment["python"], + "build_sha256": environment["build_sha256"], + } + + +def _stable_top_indices(row: np.ndarray, count: int) -> np.ndarray: + indices = np.arange(row.shape[0]) + order = np.lexsort((indices, -row)) + return order[:count] + + +def _affinity(embeddings: np.ndarray, neighbors: int) -> np.ndarray: + values = np.asarray(embeddings, dtype=np.float64) + norms = np.linalg.norm(values, axis=1) + if values.ndim != 2 or not np.isfinite(values).all() or (norms == 0).any(): + raise ValueError("Матрица эмбеддингов должна быть конечной и ненулевой") + normalized = values / norms[:, None] + similarity = normalized @ normalized.T + binary = np.zeros_like(similarity) + for row_index, row in enumerate(similarity): + binary[row_index, _stable_top_indices(row, neighbors)] = 1.0 + graph = (binary + binary.T) / 2.0 + np.fill_diagonal(graph, 0.0) + return graph + + +def _eigen_result(embeddings: np.ndarray, neighbors: int) -> dict[str, Any]: + graph = _affinity(embeddings, neighbors) + degree = np.diag(np.sum(np.abs(graph), axis=1)) + eigenvalues = np.linalg.eigvalsh(degree - graph) + max_clusters = min(8, len(eigenvalues) - 1) + gaps = np.diff(eigenvalues)[:max_clusters] + count = 1 + int(np.argmax(gaps)) + connected = int(np.count_nonzero(eigenvalues < 1e-8)) == 1 + return { + "num_clusters": count, + "max_gap": float(gaps[count - 1]), + "lambda_max": float(eigenvalues[-1]), + "connected": connected, + } + + +def estimate_eigengap(embeddings: np.ndarray, pval: float = 0.012) -> dict[str, Any]: + """Считает контрольный eigengap с фиксированным прореживанием.""" + row_count = int(np.asarray(embeddings).shape[0]) + if row_count < 2: + raise ValueError("Для eigengap нужны минимум две строки") + neighbors = max(6, round(row_count * pval)) + neighbors = min(row_count, neighbors) + return {"p": neighbors, **_eigen_result(embeddings, neighbors)} + + +def _nme_grid(row_count: int) -> list[int]: + upper = row_count // 4 + if row_count < 6 or upper < 1: + return [] + if upper <= 20: + return list(range(1, upper + 1)) + return sorted({int(value) for value in np.linspace(1, upper, 20)}) + + +def estimate_nme(embeddings: np.ndarray) -> dict[str, Any]: + """Оценивает N и p по закрепленному контракту NME Sparse-Search-20.""" + values = np.asarray(embeddings) + row_count = int(values.shape[0]) + grid = _nme_grid(row_count) + if not grid: + return {"status": "skipped", "reason": "fewer-than-six-rows"} + candidates = [] + epsilon = 1e-10 + for neighbors in grid: + result = _eigen_result(values, neighbors) + normalized_gap = result["max_gap"] / (result["lambda_max"] + epsilon) + score = (neighbors / row_count) / (normalized_gap + epsilon) + candidates.append({"p": neighbors, "score": score, **result}) + selected_index = min( + range(len(candidates)), + key=lambda index: (candidates[index]["score"], candidates[index]["p"]), + ) + while ( + selected_index < len(candidates) and not candidates[selected_index]["connected"] + ): + selected_index += 1 + if selected_index == len(candidates): + raise ValueError("NME: в сетке нет связного графа") + selected = candidates[selected_index] + return { + "status": "complete", + "p": selected["p"], + "num_clusters": selected["num_clusters"], + "score": selected["score"], + } + + +def select_counter(results: Sequence[Mapping[str, Any]]) -> dict[str, str | None]: + """Выбирает счетчик отдельно для каждой embedding-модели.""" + selected: dict[str, str | None] = {} + model_ids = sorted({str(item["embedding_id"]) for item in results}) + for model_id in model_ids: + passing = [] + for counter in ("eigengap", "nme"): + rows = [ + item + for item in results + if item["embedding_id"] == model_id and item["counter"] == counter + ] + if rows and [item["num_clusters"] for item in rows] == [3, 2, 2]: + passing.append( + (counter, sum(float(item["wall_seconds"]) for item in rows)) + ) + if not passing: + selected[model_id] = None + elif len(passing) == 1: + selected[model_id] = passing[0][0] + else: + times = dict(passing) + faster = min(times, key=times.get) + slower = max(times.values()) + difference = 0.0 if slower == 0 else (slower - min(times.values())) / slower + selected[model_id] = "eigengap" if difference <= 0.1 else faster + return selected + + +def select_working_shift(results: Sequence[Mapping[str, Any]]) -> float: + """Выбирает безопасный шаг с порогом продолжения 10%.""" + by_shift = {float(item["window_shift_ratio"]): item for item in results} + baseline = by_shift.get(0.1) + if baseline is None or not baseline.get("quality_passed"): + raise ValueError("Нет принятого W0 baseline") + eligible = [ + item + for item in results + if item.get("quality_passed") + and item.get("work_counters_explained") + and float(item["wall_seconds"]) <= float(baseline["wall_seconds"]) * 0.9 + ] + if not eligible: + return 0.1 + fastest = min(float(item["wall_seconds"]) for item in eligible) + equal = [ + item + for item in eligible + if abs(float(item["wall_seconds"]) - fastest) / fastest <= 0.05 + ] + return min(float(item["window_shift_ratio"]) for item in equal) + + +def assemble_candidate_recipes( + working_shift: float, + branches: Sequence[Mapping[str, Any]], +) -> list[dict[str, Any]]: + """Собирает C0-C2 без декартова перебора и дублей.""" + recipes: list[dict[str, Any]] = [] + if working_shift != 0.1: + recipes.append( + { + "id": "C0", + "window_shift_ratio": working_shift, + "embedding": "wespeaker-fp32", + "inference": "sequential", + } + ) + passing = [ + item + for item in branches + if item.get("quality_passed") is True + and item.get("memory_passed") is True + and item.get("diagnostic_approved") is True + and item.get("compatible", True) + ] + continuation = [item for item in passing if item.get("speedup", 0.0) >= 0.1] + if continuation: + strongest = max(continuation, key=lambda item: float(item["speedup"])) + recipes.append( + {"id": "C1", "window_shift_ratio": working_shift, **strongest["recipe"]} + ) + isa = [item for item in passing if item.get("isa_stack")] + if isa: + recipe = {"id": "C2", "window_shift_ratio": working_shift} + for item in sorted(isa, key=lambda value: value.get("stack_order", 0)): + recipe.update(item["recipe"]) + recipes.append(recipe) + unique: list[dict[str, Any]] = [] + seen: set[str] = set() + for recipe in recipes: + semantic = {key: value for key, value in recipe.items() if key != "id"} + digest = _digest_json(semantic) + if digest not in seen: + seen.add(digest) + unique.append(recipe) + return unique[:3] + + +def finalize_handoff( + manifest: Mapping[str, Any], + output: Mapping[str, Any], + candidate_names: Sequence[str], + *, + capsule_root: Path, + gate_evidence: Mapping[str, Mapping[str, Any]], + public_report_sha256: str, + capsule_id: str, + capsule_sha256: str | None = None, +) -> dict[str, Any]: + """Финализирует handoff только из завершенных и явно принятых результатов.""" + if output.get("experiment_id") != make_experiment_id(manifest): + raise ValueError("Output относится к другому experiment") + if len(candidate_names) != len(set(candidate_names)) or len(candidate_names) > 3: + raise ValueError("Handoff принимает не больше трех уникальных кандидатов") + cells_by_name = { + cell["name"]: cell + for cell in manifest["cells"] + if cell["phase"] == "combination" and cell["enabled"] + } + results_by_id = { + item.get("cell_id"): item + for item in output.get("cells", []) + if item.get("status") == "complete" + } + candidates = [] + result_files = [] + for name in candidate_names: + cell = cells_by_name.get(name) + if cell is None: + raise ValueError(f"Кандидат {name} не является enabled combination") + cell_id = make_cell_id(manifest, cell) + result_row = results_by_id.get(cell_id) + if result_row is None or not isinstance(result_row.get("result"), Mapping): + raise ValueError(f"Для кандидата {name} нет завершенного результата") + gates = gate_evidence.get(name) + if not isinstance(gates, Mapping): + raise TypeError(f"Для кандидата {name} нет решения по gates") + _exact_keys( + gates, + {"memory_passed", "diagnostic_approved"}, + f"gates:{name}", + ) + mandatory_passed = ( + result_row["result"].get("aggregate", {}).get("quality_passed") is True + ) + result_file = f"results/{cell_id}.json" + result_bytes = _canonical_json_bytes(result_row["result"]) + evidence = { + "name": name, + "cell_id": cell_id, + "result_file": result_file, + "result_sha256": hashlib.sha256(result_bytes).hexdigest(), + "mandatory_passed": mandatory_passed, + "memory_passed": gates["memory_passed"] is True, + "diagnostic_approved": gates["diagnostic_approved"] is True, + } + if not all( + evidence[key] + for key in ( + "mandatory_passed", + "memory_passed", + "diagnostic_approved", + ) + ): + raise ValueError(f"Кандидат {name} не прошел обязательные gates") + candidates.append(evidence) + result_files.append((result_file, result_bytes)) + + _require_private_location(capsule_root, "capsule-root") + for relative_path, content in result_files: + _save_text(capsule_root / relative_path, content.decode("ascii")) + result = copy.deepcopy(manifest) + result["handoff"] = { + "outcome": "handoff" if candidates else "stop-before-ryzen", + "source_commit": manifest["experiment"]["source_commit"], + "candidate_recipes": candidates, + "public_report_sha256": public_report_sha256, + "capsule_id": capsule_id, + "capsule_sha256": capsule_sha256, + } + validate_manifest(result, verify_files=False) + return result + + +def residual_cluster_metrics( + intervals: Sequence[Mapping[str, Any]], + words: Sequence[Mapping[str, Any]], + expected_speakers: int, +) -> dict[str, float]: + """Разделяет долю времени речи и долю слов в остаточных кластерах.""" + durations: dict[int, float] = {} + for interval in intervals: + speaker = int(interval["speaker"]) + durations[speaker] = durations.get(speaker, 0.0) + max( + 0.0, float(interval["end"]) - float(interval["start"]) + ) + ordered = sorted(durations, key=lambda speaker: (-durations[speaker], speaker)) + residual = set(ordered[expected_speakers:]) + total_duration = sum(durations.values()) + assigned_words = [word for word in words if word.get("speaker") is not None] + residual_words = [ + word for word in assigned_words if int(word["speaker"]) in residual + ] + return { + "residual_speech_time_share": ( + sum(durations[speaker] for speaker in residual) / total_duration + if total_duration + else 0.0 + ), + "residual_assigned_word_share": ( + len(residual_words) / len(assigned_words) if assigned_words else 0.0 + ), + } + + +def pending_cells( + manifest: Mapping[str, Any], output: Mapping[str, Any], warnings: Sequence[str] +) -> list[dict[str, Any]]: + complete = { + item["cell_id"] + for item in output.get("cells", []) + if item.get("status") == "complete" + } + qdq_available = not warnings and manifest["qdq"]["enabled"] + qdq_artifact_id = manifest["qdq"]["graph_artifact_id"] + result = [] + for cell in sorted(manifest["cells"], key=lambda item: item["schedule_position"]): + cell_id = make_cell_id(manifest, cell) + enabled = cell["enabled"] and ( + qdq_available or cell["embedding_artifact_id"] != qdq_artifact_id + ) + if enabled and cell_id not in complete: + result.append({**copy.deepcopy(cell), "cell_id": cell_id}) + return result + + +def _save_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, delete=False + ) as target: + temporary = Path(target.name) + json.dump(value, target, ensure_ascii=False, indent=2) + target.write("\n") + target.flush() + os.fsync(target.fileno()) + temporary.replace(path) + + +def _save_text(path: Path, value: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, delete=False + ) as target: + temporary = Path(target.name) + target.write(value) + target.flush() + os.fsync(target.fileno()) + temporary.replace(path) + + +def _require_private_location(path: Path, label: str) -> None: + repository_root = Path(__file__).parents[2].resolve() + resolved = path.resolve() + if not resolved.is_relative_to(repository_root): + return + ignored = subprocess.run( + ["git", "check-ignore", "--quiet", "--", str(resolved)], + cwd=repository_root, + check=False, + ) + if ignored.returncode != 0: + raise ValueError(f"{label}: приватный путь не игнорируется Git") + + +def _load_output( + path: Path, manifest: Mapping[str, Any], environment: Mapping[str, Any] +) -> dict[str, Any]: + experiment_id = make_experiment_id(manifest) + if not path.exists(): + return { + "schema_version": SCHEMA_VERSION, + "experiment_id": experiment_id, + "environment": environment, + "cells": [], + } + output = json.loads(path.read_text(encoding="utf-8")) + if ( + output.get("schema_version") != SCHEMA_VERSION + or output.get("experiment_id") != experiment_id + ): + raise ValueError("Существующий output относится к другому experiment") + if _stable_environment_identity(output.get("environment", {})) != ( + _stable_environment_identity(environment) + ): + raise ValueError("Существующий output снят в другом окружении") + return output + + +def _stable_environment_identity(environment: Mapping[str, Any]) -> dict[str, Any]: + """Исключает только значения, которые закономерно меняются между resume.""" + result = copy.deepcopy(dict(environment)) + result.pop("swap_used_bytes", None) + return result + + +def _replace_cell(output: dict[str, Any], result: Mapping[str, Any]) -> None: + output["cells"] = [ + item for item in output["cells"] if item.get("cell_id") != result["cell_id"] + ] + [dict(result)] + + +def run_schedule( + manifest: Mapping[str, Any], + output_path: Path, + work_dir: Path, + runner: Callable[[Mapping[str, Any]], Mapping[str, Any]], + *, + environment: Mapping[str, Any] | None = None, + guard_reader: Callable[[], Mapping[str, Any]] = runtime_guard_snapshot, +) -> dict[str, Any]: + """Выполняет оставшиеся ячейки и сохраняет каждый сырой повтор.""" + warnings = validate_manifest(manifest) + current_environment = dict(environment or environment_snapshot()) + validate_environment(manifest, current_environment) + output = _load_output(output_path, manifest, current_environment) + _save_json(output_path, output) + for cell in pending_cells(manifest, output, warnings): + request = { + "worker_schema_version": SCHEMA_VERSION, + "experiment_id": output["experiment_id"], + "cell": cell, + "manifest": manifest, + "work_dir": str(work_dir), + } + previous = next( + ( + item + for item in output["cells"] + if item.get("cell_id") == cell["cell_id"] + ), + {}, + ) + attempts = list(previous.get("guard", {}).get("attempts", [])) + if len(attempts) >= 2 and all(not item.get("valid") for item in attempts): + raise RuntimeError(f"Ячейка {cell['name']} исчерпала retry runtime guard") + result = None + for attempt_number in range(len(attempts) + 1, 3): + before = dict(guard_reader()) + payload = None + error = None + try: + payload = dict(runner(request)) + except Exception as exc: # noqa: BLE001 — граница изолированного worker + error = exc + after = dict(guard_reader()) + reasons = validate_runtime_guard_transition(manifest, before, after) + attempt = { + "attempt": attempt_number, + "before": before, + "after": after, + "valid": not reasons, + "reasons": reasons, + } + attempts.append(attempt) + if error is not None: + result = { + "cell_id": cell["cell_id"], + "status": "failed", + "error_type": type(error).__name__, + "guard": {"attempts": attempts}, + } + break + if reasons: + result = { + "cell_id": cell["cell_id"], + "status": "retrying" if attempt_number == 1 else "failed", + "error_type": "RuntimeGuardError", + "guard": {"attempts": attempts}, + } + _replace_cell(output, result) + _save_json(output_path, output) + if attempt_number == 1: + continue + break + result = { + "cell_id": cell["cell_id"], + "status": "complete", + "result": payload, + "guard": {"attempts": attempts}, + } + break + if result is None: + raise AssertionError("Runtime guard не сформировал результат") + _replace_cell(output, result) + _save_json(output_path, output) + if result["status"] != "complete": + raise RuntimeError(f"Ячейка {cell['name']} завершилась с ошибкой") + return output + + +def subprocess_runner(request: Mapping[str, Any]) -> Mapping[str, Any]: + """Запускает ячейку в новом процессе без публикации приватного stderr.""" + work_dir = Path(request["work_dir"]) + request_dir = work_dir / "requests" + request_dir.mkdir(parents=True, exist_ok=True) + request_path = request_dir / f"{request['cell']['cell_id']}.json" + response_path = request_dir / f"{request['cell']['cell_id']}.result.json" + payload = dict(request) + payload["response_path"] = str(response_path) + _save_json(request_path, payload) + worker_python = os.environ.get( + "LOCAL_TRANSCRIBER_EXPERIMENT_PYTHON", sys.executable + ) + completed = subprocess.run( + [ + worker_python, + str( + Path(__file__).parents[2] + / "scripts" + / "benchmarks" + / "diarization_calibration.py" + ), + "--worker", + str(request_path), + ], + capture_output=True, + text=True, + check=False, + ) + log_dir = work_dir / "logs" + _save_text(log_dir / f"{request['cell']['cell_id']}.stdout.log", completed.stdout) + _save_text(log_dir / f"{request['cell']['cell_id']}.stderr.log", completed.stderr) + if completed.returncode != 0 or not response_path.is_file(): + raise RuntimeError("Дочерний процесс завершился с ошибкой") + return json.loads(response_path.read_text(encoding="utf-8")) + + +def worker_main(request_path: Path) -> None: + request = json.loads(request_path.read_text(encoding="utf-8")) + from local_transcriber.experimental_diarization import run_cell + + result = run_cell(request) + _save_json(Path(request["response_path"]), result) + + +def verify_capsule( + archive_path: Path, + expected_sha256: str, + *, + expected_manifest_path: Path, + expected_source_commit: str | None = None, + expected_report_sha256: str | None = None, + expected_capsule_id: str | None = None, +) -> dict[str, Any]: + """Сверяет внешний и внутренние хеши приватной ZIP-капсулы.""" + if file_sha256(archive_path) != expected_sha256: + raise ValueError("SHA-256 ZIP-капсулы не совпал") + with zipfile.ZipFile(archive_path) as archive: + names = archive.namelist() + if len(names) != len(set(names)): + raise ValueError("Капсула содержит повторяющиеся пути") + for name in names: + path = PurePosixPath(name) + if path.is_absolute() or ".." in path.parts or "\\" in name: + raise ValueError("Капсула содержит небезопасный путь") + if "capsule.json" not in names: + raise ValueError("В капсуле нет capsule.json") + capsule = json.loads(archive.read("capsule.json")) + _exact_keys( + capsule, + {"capsule_id", "source_commit", "public_report_sha256", "files"}, + "capsule.json", + ) + _require_id(capsule["capsule_id"], "capsule.json.capsule_id") + _require_commit(capsule["source_commit"], "capsule.json.source_commit") + _require_sha256( + capsule["public_report_sha256"], + "capsule.json.public_report_sha256", + ) + if expected_capsule_id and capsule["capsule_id"] != expected_capsule_id: + raise ValueError("Идентификатор капсулы не совпал") + if ( + expected_source_commit + and capsule.get("source_commit") != expected_source_commit + ): + raise ValueError("Исходный commit капсулы не совпал") + if ( + expected_report_sha256 + and capsule.get("public_report_sha256") != expected_report_sha256 + ): + raise ValueError("Хеш публичного отчета в капсуле не совпал") + declared = capsule.get("files") + if not isinstance(declared, dict): + raise TypeError("capsule.json: нужен словарь files") + if "manifest.json" not in declared or "manifest.json" not in names: + raise ValueError("Капсула должна содержать корневой manifest.json") + actual_files = {name for name in names if not name.endswith("/")} + if actual_files != {"capsule.json", *declared}: + raise ValueError("capsule.json должен перечислять каждый файл архива") + for name, expected in declared.items(): + if name == "capsule.json" or name not in names: + raise ValueError("capsule.json ссылается на отсутствующий файл") + actual = hashlib.sha256(archive.read(name)).hexdigest() + if actual != expected: + raise ValueError("Внутренний SHA-256 капсулы не совпал") + manifest_bytes = expected_manifest_path.read_bytes() + archived_manifest = archive.read("manifest.json") + if archived_manifest != manifest_bytes: + raise ValueError("manifest аргумента не совпал с manifest.json капсулы") + manifest_sha256 = hashlib.sha256(manifest_bytes).hexdigest() + if declared["manifest.json"] != manifest_sha256: + raise ValueError("Хеш manifest.json не совпал с индексом капсулы") + archived_manifest_value = json.loads(archived_manifest) + handoff = archived_manifest_value.get("handoff", {}) + for candidate in handoff.get("candidate_recipes", []): + result_file = candidate.get("result_file") + result_sha256 = candidate.get("result_sha256") + if declared.get(result_file) != result_sha256: + raise ValueError( + "Результат кандидата не совпал с индексом файлов капсулы" + ) + return { + "capsule_id": capsule.get("capsule_id"), + "source_commit": capsule.get("source_commit"), + "public_report_sha256": capsule.get("public_report_sha256"), + "file_count": len(declared), + "manifest_sha256": manifest_sha256, + } + + +def run_cli(args: Any, raw_manifest: Mapping[str, Any]) -> None: + _require_private_location(args.manifest, "manifest") + if args.output is not None: + _require_private_location(args.output, "output") + if args.work_dir is not None: + _require_private_location(args.work_dir, "work-dir") + validate_manifest( + raw_manifest, + verify_files=False, + require_relative_paths=True, + ) + manifest = resolve_manifest_paths(raw_manifest, args.manifest.parent) + warnings = validate_manifest(manifest) + environment = environment_snapshot() + validate_environment(manifest, environment) + capsule_path = getattr(args, "capsule", None) + capsule_sha256 = getattr(args, "capsule_sha256", None) + finalized_handoff = manifest["handoff"]["outcome"] != "pending" + if finalized_handoff and (capsule_path is None or capsule_sha256 is None): + raise ValueError("Итоговый handoff требует --capsule и --capsule-sha256") + capsule_result = None + if capsule_path is not None and capsule_sha256 is not None: + capsule_result = verify_capsule( + capsule_path, + capsule_sha256, + expected_manifest_path=args.manifest, + expected_source_commit=manifest["handoff"]["source_commit"], + expected_report_sha256=manifest["handoff"]["public_report_sha256"], + expected_capsule_id=manifest["handoff"]["capsule_id"], + ) + if args.validate_only: + plan = safe_plan(manifest, warnings, environment) + if capsule_result is not None: + plan["capsule"] = capsule_result + print(json.dumps(plan, ensure_ascii=False, indent=2)) + return + if args.prepare_asr is not None: + raise ValueError("manifest v3 не поддерживает --prepare-asr") + args.work_dir.mkdir(parents=True, exist_ok=True) + run_schedule( + manifest, args.output, args.work_dir, subprocess_runner, environment=environment + ) + + +def synthetic_block_embeddings( + cluster_sizes: Iterable[int], + *, + dimensions: int | None = None, + seed: int = 0, + noise: float = 0.0, +) -> np.ndarray: + """Строит детерминированную фикстуру для дешевого smoke-test счетчиков.""" + sizes = tuple(cluster_sizes) + random = np.random.default_rng(seed) + dimension = dimensions or len(sizes) + centers = random.normal(size=(len(sizes), dimension)) + centers /= np.linalg.norm(centers, axis=1)[:, None] + rows = [] + for cluster, size in enumerate(sizes): + rows.extend( + centers[cluster] + random.normal(0.0, noise, dimension) for _ in range(size) + ) + return np.asarray(rows) diff --git a/src/local_transcriber/experimental_diarization.py b/src/local_transcriber/experimental_diarization.py new file mode 100644 index 0000000..4005478 --- /dev/null +++ b/src/local_transcriber/experimental_diarization.py @@ -0,0 +1,1489 @@ +"""Исполнитель ячеек исследовательского CPU-стенда.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import tempfile +import threading +import time +from collections.abc import Callable, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import numpy as np + +from .benchmark_experiment import ( + estimate_eigengap, + estimate_nme, + residual_cluster_metrics, +) +from .diarization import _assign_cluster, build_speaker_transcript +from .types import SpeakerInterval, Word + +SAMPLE_RATE = 16_000 +PYANNOTE_WINDOW_SAMPLES = 160_000 +NUMBER_PATTERN = r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)" +STAGE_RE = re.compile( + rf"OfflineSpeakerDiarization:\s+(segmentation|embedding|clustering)\s+" + rf"({NUMBER_PATTERN})\s+s" +) +TOTAL_RE = re.compile( + rf"OfflineSpeakerDiarization:\s+total\s+({NUMBER_PATTERN})\s+s,\s+" + rf"audio\s+({NUMBER_PATTERN})\s+s,\s+RTF\s+({NUMBER_PATTERN})" +) +TURN_RE = re.compile(r"^\*\*\[(\d{2}):(\d{2})(?::(\d{2}))?\] Speaker (\d+):\*\*") + + +class PeakRssSampler: + """Сэмплирует RSS только вокруг измеряемого вызова.""" + + def __init__(self, process: Any, interval_seconds: float): + self._process = process + self._interval_seconds = interval_seconds + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self.peak_bytes = 0 + + def _sample(self) -> None: + self.peak_bytes = max(self.peak_bytes, int(self._process.memory_info().rss)) + + def _run(self) -> None: + while not self._stop.wait(self._interval_seconds): + self._sample() + + def start(self) -> None: + self._sample() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self) -> int: + self._stop.set() + if self._thread is not None: + self._thread.join() + self._sample() + return self.peak_bytes + + +def segmentation_window_count(sample_count: int, shift_samples: int) -> int: + """Повторяет схему полного и последнего дополненного окна sherpa.""" + if sample_count <= 0 or shift_samples <= 0: + raise ValueError("Число сэмплов и шаг должны быть положительными") + if sample_count <= PYANNOTE_WINDOW_SAMPLES: + return 1 + full = (sample_count - PYANNOTE_WINDOW_SAMPLES) // shift_samples + 1 + covered = (full - 1) * shift_samples + PYANNOTE_WINDOW_SAMPLES + return full + int(covered < sample_count) + + +def make_config( + cell: Mapping[str, Any], + artifacts: Mapping[str, Mapping[str, Any]], + provider_config_path: Path | None = None, +) -> Any: + """Создаёт конфигурацию 1.13.6 для одной исследовательской ячейки.""" + import sherpa_onnx + + inference = cell["inference"] + pyannote_options = { + "model": str(artifacts[cell["segmentation_artifact_id"]]["path"]), + } + version = tuple(int(part) for part in sherpa_onnx.__version__.split(".")) + if version >= (1, 13, 6): + pyannote_options["window_shift_ratio"] = float(cell["window_shift_ratio"]) + elif not math.isclose(float(cell["window_shift_ratio"]), 0.1): + raise RuntimeError("sherpa-onnx 1.13.5 поддерживает только шаг 0.1") + provider = "cpu" if provider_config_path is None else f"cpu:{provider_config_path}" + segmentation = sherpa_onnx.OfflineSpeakerSegmentationModelConfig( + pyannote=sherpa_onnx.OfflineSpeakerSegmentationPyannoteModelConfig( + **pyannote_options, + ), + num_threads=int(inference["intra_op_threads"]), + provider=provider, + debug=True, + ) + embedding = sherpa_onnx.SpeakerEmbeddingExtractorConfig( + model=str(artifacts[cell["embedding_artifact_id"]]["path"]), + num_threads=int(inference["intra_op_threads"]), + provider=provider, + debug=True, + ) + clustering = cell["clustering"] + num_clusters = clustering["num_clusters"] + return sherpa_onnx.OfflineSpeakerDiarizationConfig( + segmentation=segmentation, + embedding=embedding, + clustering=sherpa_onnx.FastClusteringConfig( + num_clusters=-1 if num_clusters is None else int(num_clusters), + threshold=float(clustering["threshold"]), + ), + min_duration_on=0.3, + min_duration_off=0.5, + ) + + +def parse_stage_timings(log: str) -> dict[str, float]: + """Разбирает штатные агрегаты debug-профиля 1.13.6.""" + stages: dict[str, list[float]] = {} + for name, raw_value in STAGE_RE.findall(log): + stages.setdefault(name, []).append(float(raw_value)) + total_matches = TOTAL_RE.findall(log) + if set(stages) != {"segmentation", "embedding", "clustering"}: + raise ValueError("Debug-журнал не содержит все стадии") + if any(len(values) != 1 for values in stages.values()) or len(total_matches) != 1: + raise ValueError("Debug-журнал содержит дубли стадий") + total, audio, rtf = map(float, total_matches[0]) + values = { + "segmentation_seconds": stages["segmentation"][0], + "embedding_seconds": stages["embedding"][0], + "clustering_seconds": stages["clustering"][0], + "engine_total_seconds": total, + "engine_audio_seconds": audio, + "engine_rtf": rtf, + } + if any(not math.isfinite(value) or value < 0 for value in values.values()): + raise ValueError("Debug-журнал содержит недопустимое время") + return values + + +def parse_prepare_timings(log: str) -> dict[str, float]: + """Разбирает две стадии экспериментального вызова prepare.""" + stages: dict[str, list[float]] = {} + for name, raw_value in STAGE_RE.findall(log): + stages.setdefault(name, []).append(float(raw_value)) + if set(stages) != {"segmentation", "embedding"}: + raise ValueError("Debug-журнал prepare не содержит обе стадии") + if any(len(values) != 1 for values in stages.values()): + raise ValueError("Debug-журнал prepare содержит дубли стадий") + result = { + "segmentation_seconds": stages["segmentation"][0], + "embedding_seconds": stages["embedding"][0], + } + if any(not math.isfinite(value) or value < 0 for value in result.values()): + raise ValueError("Debug-журнал prepare содержит недопустимое время") + return result + + +@contextmanager +def capture_native_stderr() -> Any: + """Перехватывает C/C++ stderr в приватный временный файл.""" + saved_fd = os.dup(2) + with tempfile.TemporaryFile(mode="w+b") as stream: + try: + os.dup2(stream.fileno(), 2) + yield stream + finally: + os.dup2(saved_fd, 2) + os.close(saved_fd) + + +def measure_call( + operation: Callable[[], Any], process: Any, interval_seconds: float +) -> tuple[Any, dict[str, float | int]]: + """Измеряет wall/CPU/RSS одного process-вызова.""" + before = process.cpu_times() + sampler = PeakRssSampler(process, interval_seconds) + sampler.start() + started = time.perf_counter() + try: + result = operation() + finally: + wall = time.perf_counter() - started + peak = sampler.stop() + after = process.cpu_times() + user = float(after.user - before.user) + system = float(after.system - before.system) + total = user + system + return result, { + "wall_seconds": wall, + "user_cpu_seconds": user, + "system_cpu_seconds": system, + "total_cpu_seconds": total, + "average_cpu_cores": total / wall if wall else 0.0, + "peak_rss_bytes": peak, + } + + +def _load_words(path: Path, duration: float) -> list[Word]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, list) or not raw: + raise ValueError("ASR sidecar должен быть непустым JSON-массивом") + words = [] + previous = -math.inf + for index, item in enumerate(raw): + if not isinstance(item, dict) or set(item) != {"start", "end", "text"}: + raise ValueError(f"ASR sidecar[{index}]: неверная структура") + start = float(item["start"]) + end = float(item["end"]) + if not 0 <= start <= end <= duration or start < previous: + raise ValueError(f"ASR sidecar[{index}]: неверные временные границы") + if not isinstance(item["text"], str): + raise TypeError(f"ASR sidecar[{index}].text: нужна строка") + words.append(Word(start=start, end=end, text=item["text"])) + previous = start + return words + + +def _text_hash(words: Sequence[Word]) -> str: + normalized = "".join("".join(word.text.split()) for word in words) + return hashlib.sha256(normalized.encode()).hexdigest() + + +def _transcript_text_hash(transcript: Any) -> str: + normalized = "".join("".join(turn.text.split()) for turn in transcript.turns) + return hashlib.sha256(normalized.encode()).hexdigest() + + +def _timestamp_seconds(match: re.Match[str]) -> float: + first, second, third = match.group(1), match.group(2), match.group(3) + if third is None: + return int(first) * 60 + int(second) + return int(first) * 3600 + int(second) * 60 + int(third) + + +def _read_reference_turns(recording: Mapping[str, Any]) -> list[dict[str, Any]]: + reference = recording["reference"] + if reference is None: + return [] + starts = [] + for line in Path(reference["path"]).read_text(encoding="utf-8").splitlines(): + match = TURN_RE.match(line) + if match: + starts.append((_timestamp_seconds(match), match.group(4))) + clip_start = float(recording["start"]) + clip_end = clip_start + float(recording["duration"]) + turns = [] + for index, (start, speaker) in enumerate(starts): + end = starts[index + 1][0] if index + 1 < len(starts) else clip_end + overlap_start = max(start, clip_start) + overlap_end = min(end, clip_end) + if overlap_end > overlap_start: + turns.append( + { + "speaker": speaker, + "start": overlap_start - clip_start, + "end": overlap_end - clip_start, + } + ) + return turns + + +def _mapped_speaker_purity( + segments: Sequence[Mapping[str, Any]], + reference_turns: Sequence[Mapping[str, Any]], +) -> float | None: + import itertools + from collections import defaultdict + + if not segments or not reference_turns: + return None + predicted = sorted({str(item["speaker"]) for item in segments}) + reference = sorted({str(item["speaker"]) for item in reference_turns}) + overlap: dict[tuple[str, str], float] = defaultdict(float) + total = 0.0 + for segment in segments: + for turn in reference_turns: + value = max( + 0.0, + min(float(segment["end"]), float(turn["end"])) + - max(float(segment["start"]), float(turn["start"])), + ) + overlap[(str(segment["speaker"]), str(turn["speaker"]))] += value + total += value + if len(predicted) >= len(reference): + candidates = ( + zip(choice, reference, strict=True) + for choice in itertools.permutations(predicted, len(reference)) + ) + else: + candidates = ( + zip(predicted, choice, strict=True) + for choice in itertools.permutations(reference, len(predicted)) + ) + best = max(sum(overlap[pair] for pair in candidate) for candidate in candidates) + return best / total if total else None + + +def _decode_clip(recording: Mapping[str, Any]) -> np.ndarray: + from faster_whisper import decode_audio + + samples = decode_audio(str(recording["path"]), sampling_rate=SAMPLE_RATE) + if isinstance(samples, tuple): + raise TypeError("Декодер вернул раздельные стереоканалы") + start = round(float(recording["start"]) * SAMPLE_RATE) + length = round(float(recording["duration"]) * SAMPLE_RATE) + clip = np.asarray(samples[start : start + length], dtype=np.float32) + if len(clip) != length: + raise ValueError("Запись короче объявленного интервала") + return clip + + +def _make_progress_callback() -> tuple[Callable[[int, int], int], dict[str, Any]]: + state: dict[str, Any] = {"total": 0, "seen": set()} + + def callback(processed: int, total: int) -> int: + state["total"] = max(state["total"], int(total)) + state["seen"].add(int(processed)) + return 0 + + return callback, state + + +def _write_provider_config(cell: Mapping[str, Any], work_dir: Path) -> Path: + inference = cell["inference"] + directory = work_dir / "provider-configs" + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{cell['cell_id']}.config" + path.write_text( + "\n".join( + ( + f"IntraOpNumThreads={int(inference['intra_op_threads'])}", + f"InterOpNumThreads={int(inference['inter_op_threads'])}", + "ExecutionMode=0", + "", + ) + ), + encoding="utf-8", + ) + return path + + +def _cluster_prepared( + sherpa_onnx: Any, + prepared: Any, + cell: Mapping[str, Any], + expected_speakers: int, + embeddings_override: np.ndarray | None = None, +) -> tuple[np.ndarray, dict[str, Any], float]: + source = prepared.embeddings if embeddings_override is None else embeddings_override + embeddings = np.ascontiguousarray(np.asarray(source, dtype=np.float32)) + if prepared.is_terminal: + return np.empty(0, dtype=np.int32), {"status": "terminal"}, 0.0 + if embeddings.ndim != 2 or not np.isfinite(embeddings).all(): + raise ValueError("Prepare вернул недопустимую матрицу эмбеддингов") + + counter_started = time.perf_counter() + counter = cell["counter"] + if counter == "eigengap": + counter_result = estimate_eigengap(embeddings.copy()) + num_clusters = int(counter_result["num_clusters"]) + elif counter == "nme": + counter_result = estimate_nme(embeddings.copy()) + if counter_result.get("status") != "complete": + raise ValueError("NME не смог выбрать число кластеров") + num_clusters = int(counter_result["num_clusters"]) + elif counter == "known": + num_clusters = expected_speakers + counter_result = {"status": "known", "num_clusters": num_clusters} + elif counter == "threshold": + num_clusters = -1 + counter_result = { + "status": "threshold", + "threshold": float(cell["clustering"]["threshold"]), + } + else: + raise ValueError("Неизвестный счетчик говорящих") + counter_seconds = time.perf_counter() - counter_started + + clustering = sherpa_onnx.FastClustering( + sherpa_onnx.FastClusteringConfig( + num_clusters=num_clusters, + threshold=float(cell["clustering"]["threshold"]), + ) + ) + cluster_started = time.perf_counter() + labels = np.asarray(clustering(embeddings.copy()), dtype=np.int32) + cluster_seconds = time.perf_counter() - cluster_started + counter_result = { + **counter_result, + "seconds": counter_seconds, + "row_count": int(embeddings.shape[0]), + } + return labels, counter_result, cluster_seconds + + +def _validate_cluster_repeatability( + sherpa_onnx: Any, + prepared: Any, + cell: Mapping[str, Any], + expected_speakers: int, + labels: np.ndarray, + counter_result: Mapping[str, Any], + embeddings_override: np.ndarray | None = None, +) -> dict[str, Any]: + """Повторяет счетчик и кластеризацию вне публикуемого измерения.""" + repeated_labels, repeated_counter, _ = _cluster_prepared( + sherpa_onnx, + prepared, + cell, + expected_speakers, + embeddings_override, + ) + repeated_keys = {"status", "p", "num_clusters", "threshold"} + repeatable_parameters = all( + counter_result.get(key) == repeated_counter.get(key) + for key in repeated_keys + if key in counter_result or key in repeated_counter + ) + if not repeatable_parameters: + raise RuntimeError("Счетчик дал неповторяемые p или N на одной матрице") + repeatable_labels = np.array_equal(labels, repeated_labels) + if not repeatable_labels: + raise RuntimeError("FastClustering дал неповторяемые метки на одной матрице") + return { + **counter_result, + "repeatable_parameters": True, + "repeatable_labels": True, + } + + +def _validation_metrics(measured: Mapping[str, Any]) -> dict[str, Any]: + """Отделяет стоимость и RSS обязательных проверок от результата ячейки.""" + return {f"validation_{name}": value for name, value in measured.items()} + + +def _validate_special_inference( + cell: Mapping[str, Any], physical_cores: int +) -> None: + """Повторно проверяет бюджеты специальных путей внутри worker-процесса.""" + inference = cell["inference"] + mode = inference["mode"] + outer = int(inference["outer_workers"]) + sessions = int(inference["session_count"]) + intra = int(inference["intra_op_threads"]) + inter = int(inference["inter_op_threads"]) + batch = int(inference["batch_size"]) + if min(outer, sessions, intra, inter, batch, physical_cores) <= 0: + raise ValueError("Бюджеты специального runner должны быть положительными") + if inter != 1: + raise ValueError("Специальный runner требует inter_op_threads=1") + if max(outer, sessions * intra) > physical_cores: + raise ValueError("Специальный runner превышает бюджет физических ядер") + if mode == "separate-session": + if ( + outer != physical_cores + or sessions != physical_cores + or intra != 1 + or batch != 1 + ): + raise ValueError( + "separate-session требует P worker, P session, intra-op=1 и batch=1" + ) + elif mode == "titanet-batch": + if ( + outer != 1 + or sessions != 1 + or intra != physical_cores + or batch not in {1, 4} + ): + raise ValueError( + "titanet-batch требует одну session, один worker, intra-op=P " + "и batch 1 или 4" + ) + if cell["counter"] != "nme": + raise ValueError("titanet-batch проверяется только со счетчиком NME") + else: + raise ValueError("Ожидался специальный inference mode") + + +def _validate_embedding_rows(embeddings: np.ndarray, expected_rows: int) -> None: + if embeddings.ndim != 2 or embeddings.shape[0] != expected_rows: + raise RuntimeError("Экспериментальный inference потерял строки эмбеддингов") + if not np.isfinite(embeddings).all(): + raise RuntimeError("Экспериментальный inference вернул нечисловые значения") + + +def _prepared_rows_preserved(prepared: Any) -> bool: + counters = dict(prepared.counters) + row_count = len(prepared.row_mapping) + required = { + "embedding_jobs", + "embedding_run_calls", + "embedding_rows_dropped", + } + if not required <= counters.keys(): + return False + try: + embedding_jobs = int(counters["embedding_jobs"]) + embedding_run_calls = int(counters["embedding_run_calls"]) + embedding_rows_dropped = int(counters["embedding_rows_dropped"]) + return ( + embedding_jobs == row_count + and embedding_run_calls >= 0 + and embedding_rows_dropped == 0 + ) + except (TypeError, ValueError): + return False + + +def _optional_work_counter(counters: Mapping[str, Any], name: str) -> int | None: + """Не подставляет правдоподобное значение вместо отсутствующего счетчика.""" + try: + return int(counters[name]) + except (KeyError, TypeError, ValueError): + return None + + +def _compute_separate_embeddings( + extractors: Sequence[Any], + row_mapping: Sequence[Any], + samples: np.ndarray, + outer_workers: int, +) -> np.ndarray: + """Вычисляет строки независимыми session и восстанавливает исходный порядок.""" + if len(extractors) != outer_workers: + raise ValueError("Число extractor должно совпадать с outer_workers") + partitions = [ + list(range(worker, len(row_mapping), outer_workers)) + for worker in range(outer_workers) + ] + + def run_partition(extractor: Any, indices: Sequence[int]) -> list[tuple[int, Any]]: + rows = [] + for index in indices: + stream = extractor.create_stream() + intervals = row_mapping[index][2] + if not intervals: + raise ValueError("Строка эмбеддинга не содержит аудиоинтервалов") + for start, end in intervals: + if not 0 <= start < end: + raise ValueError("Prepare вернул неверный аудиоинтервал") + stream.accept_waveform( + SAMPLE_RATE, + samples[int(start) : min(int(end), len(samples))], + ) + stream.input_finished() + if not extractor.is_ready(stream): + raise RuntimeError("Embedding stream не готов к вычислению") + rows.append((index, extractor.compute(stream))) + return rows + + with ThreadPoolExecutor(max_workers=outer_workers) as pool: + futures = [ + pool.submit(run_partition, extractor, indices) + for extractor, indices in zip(extractors, partitions, strict=True) + ] + indexed = [item for future in futures for item in future.result()] + indexed.sort(key=lambda item: item[0]) + if [index for index, _ in indexed] != list(range(len(row_mapping))): + raise RuntimeError("Отдельные session потеряли или продублировали строки") + embeddings = np.asarray([row for _, row in indexed], dtype=np.float32) + _validate_embedding_rows(embeddings, len(row_mapping)) + return embeddings + + +def _titanet_features(waveform: np.ndarray) -> np.ndarray: + """Извлекает признаки по контракту официального TitaNet small.""" + import kaldi_native_fbank as knf + + options = knf.FbankOptions() + options.frame_opts.samp_freq = SAMPLE_RATE + options.frame_opts.dither = 0.0 + options.frame_opts.snip_edges = True + options.frame_opts.frame_shift_ms = 10.0 + options.frame_opts.frame_length_ms = 25.0 + options.frame_opts.remove_dc_offset = False + options.frame_opts.preemph_coeff = 0.97 + options.frame_opts.window_type = "hann" + options.frame_opts.round_to_power_of_two = True + options.mel_opts.num_bins = 80 + options.mel_opts.low_freq = 0.0 + options.mel_opts.high_freq = -400.0 + options.mel_opts.is_librosa = True + extractor = knf.OnlineFbank(options) + extractor.accept_waveform(SAMPLE_RATE, waveform.tolist()) + extractor.input_finished() + features = np.asarray( + [extractor.get_frame(index) for index in range(extractor.num_frames_ready)], + dtype=np.float32, + ) + if features.ndim != 2 or not len(features) or not np.isfinite(features).all(): + raise RuntimeError("Не удалось получить конечные признаки TitaNet") + mean = features.mean(axis=0, keepdims=True) + variance = np.square(features - mean).mean(axis=0, keepdims=True) + return (features - mean) / (np.sqrt(variance) + np.float32(1e-5)) + + +def _prepare_titanet_features( + row_mapping: Sequence[Any], samples: np.ndarray +) -> list[np.ndarray]: + features = [] + for row in row_mapping: + intervals = row[2] + if not intervals: + raise ValueError("Строка TitaNet не содержит аудиоинтервалов") + chunks = [ + samples[int(start) : min(int(end), len(samples))] + for start, end in intervals + if 0 <= start < end + ] + if len(chunks) != len(intervals) or any(not len(chunk) for chunk in chunks): + raise ValueError("Prepare вернул неверный аудиоинтервал TitaNet") + features.append(_titanet_features(np.concatenate(chunks))) + if len({len(item) for item in features}) < 2: + raise RuntimeError("TitaNet batch требует входы переменной длины") + return features + + +def _infer_titanet_batches( + session: Any, features: Sequence[np.ndarray], batch_size: int +) -> np.ndarray: + """Выполняет TitaNet batch без изменения порядка строк.""" + rows = [] + for offset in range(0, len(features), batch_size): + batch = features[offset : offset + batch_size] + lengths = np.asarray([len(item) for item in batch], dtype=np.int64) + values = np.zeros((len(batch), 80, int(lengths.max())), dtype=np.float32) + for index, item in enumerate(batch): + if item.ndim != 2 or item.shape[1] != 80: + raise ValueError("Признаки TitaNet должны иметь форму [T, 80]") + values[index, :, : len(item)] = item.T + output = session.run( + ["embs"], {"audio_signal": values, "length": lengths} + )[0] + rows.extend(output) + embeddings = np.asarray(rows, dtype=np.float32) + _validate_embedding_rows(embeddings, len(features)) + return embeddings + + +def _segments_signature(diarization: Any) -> tuple[tuple[int, float, float], ...]: + return tuple( + (int(item.speaker), float(item.start), float(item.end)) + for item in diarization.sort_by_start_time() + ) + + +def _composed_metrics( + measured: Mapping[str, Any], composed_wall_seconds: float +) -> dict[str, Any]: + """Помечает wall-оценку составного экспериментального pipeline.""" + if not math.isfinite(composed_wall_seconds) or composed_wall_seconds < 0: + raise ValueError("Составной wall-time должен быть конечным и неотрицательным") + return { + **measured, + "wall_seconds": composed_wall_seconds, + "measurement_kind": "composed-estimate", + "wall_seconds_is_composed": True, + "instrumented_pipeline_wall_seconds": measured["wall_seconds"], + } + + +def _run_prepared_recording( + engine: Any, + samples: np.ndarray, + cell: Mapping[str, Any], + recording: Mapping[str, Any], + interval_seconds: float, +) -> tuple[Any, dict[str, Any], dict[str, Any], Mapping[str, Any]]: + import psutil + import sherpa_onnx + + if cell["inference"]["mode"] not in {"sequential", "shared-session"}: + raise RuntimeError( + "Этот inference mode требует отдельного экспериментального runner" + ) + operation_state: dict[str, Any] = {} + + def operation() -> Any: + prepared = engine.prepare( + samples, + int(cell["inference"]["outer_workers"]), + ) + labels, counter_result, cluster_seconds = _cluster_prepared( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + ) + finalized_started = time.perf_counter() + result = engine.finalize(prepared, labels) + operation_state.update( + { + "prepared": prepared, + "labels": labels, + "counter": counter_result, + "clustering_seconds": cluster_seconds, + "finalize_seconds": time.perf_counter() - finalized_started, + } + ) + return result + + with capture_native_stderr() as native_log: + diarization, metrics = measure_call( + operation, + psutil.Process(os.getpid()), + interval_seconds, + ) + native_log.flush() + native_log.seek(0) + prepare_metrics = parse_prepare_timings( + native_log.read().decode(errors="replace") + ) + prepared = operation_state["prepared"] + validation_state: dict[str, Any] = {} + + def validate() -> None: + validation_state["counter"] = _validate_cluster_repeatability( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + operation_state["labels"], + operation_state["counter"], + ) + validation_passed = _prepared_rows_preserved(prepared) + if cell["inference"]["mode"] == "shared-session": + baseline = engine.prepare(samples, 1) + baseline_labels, _, _ = _cluster_prepared( + sherpa_onnx, + baseline, + cell, + int(recording["expected_speakers"]), + ) + baseline_diarization = engine.finalize(baseline, baseline_labels) + rows_equal = tuple(prepared.row_mapping) == tuple(baseline.row_mapping) + candidate_embeddings = np.asarray(prepared.embeddings, dtype=np.float32) + baseline_embeddings = np.asarray(baseline.embeddings, dtype=np.float32) + finite_embeddings = bool( + np.isfinite(candidate_embeddings).all() + and np.isfinite(baseline_embeddings).all() + ) + labels_equal = np.array_equal(operation_state["labels"], baseline_labels) + final_equal = _segments_signature(diarization) == _segments_signature( + baseline_diarization + ) + validation_passed = bool( + validation_passed + and rows_equal + and finite_embeddings + and labels_equal + and final_equal + ) + validation_state["inference"] = { + "mode": "shared-session", + "rows_equal_to_sequential": rows_equal, + "finite_embeddings": finite_embeddings, + "labels_equal_to_sequential": labels_equal, + "final_equal_to_sequential": final_equal, + **_embedding_drift(candidate_embeddings, baseline_embeddings), + } + validation_state["passed"] = validation_passed + + with capture_native_stderr(): + _, validation_measurement = measure_call( + validate, + psutil.Process(os.getpid()), + interval_seconds, + ) + metrics = {**metrics, **_validation_metrics(validation_measurement)} + stage_metrics = { + **prepare_metrics, + "clustering_seconds": operation_state["clustering_seconds"], + "finalize_seconds": operation_state["finalize_seconds"], + "engine_total_seconds": metrics["wall_seconds"], + "engine_audio_seconds": len(samples) / SAMPLE_RATE, + "engine_rtf": metrics["wall_seconds"] / (len(samples) / SAMPLE_RATE), + } + prepared_data = { + "counters": dict(prepared.counters), + "counter": validation_state["counter"], + "row_mapping_count": len(prepared.row_mapping), + "row_mapping": tuple(prepared.row_mapping), + "row_labels": operation_state["labels"], + "embeddings": np.asarray(prepared.embeddings, dtype=np.float32), + "validation_passed": validation_state["passed"], + } + if "inference" in validation_state: + prepared_data["inference"] = validation_state["inference"] + return diarization, metrics, stage_metrics, prepared_data + + +def _embedding_drift( + candidate: np.ndarray, baseline: np.ndarray +) -> dict[str, float | None]: + if candidate.shape != baseline.shape: + raise RuntimeError("Контрольная и экспериментальная матрицы имеют разную форму") + denominator = np.linalg.norm(candidate, axis=1) * np.linalg.norm(baseline, axis=1) + valid = denominator > 0 + cosine = np.sum(candidate[valid] * baseline[valid], axis=1) / denominator[valid] + return { + "max_abs_drift": float(np.max(np.abs(candidate - baseline))), + "min_cosine": float(np.min(cosine)) if len(cosine) else None, + } + + +def _run_separate_session_recording( + engine: Any, + extractors: Sequence[Any], + samples: np.ndarray, + cell: Mapping[str, Any], + recording: Mapping[str, Any], + interval_seconds: float, +) -> tuple[Any, dict[str, Any], dict[str, Any], Mapping[str, Any]]: + import psutil + import sherpa_onnx + + inference = cell["inference"] + session_count = int(inference["session_count"]) + operation_state: dict[str, Any] = {} + + def operation() -> Any: + prepare_started = time.perf_counter() + prepared = engine.prepare(samples, 1) + prepare_wall = time.perf_counter() - prepare_started + if prepared.is_terminal: + raise RuntimeError("separate-session не получил строки эмбеддингов") + mapping = tuple(prepared.row_mapping) + + def compute_embeddings() -> np.ndarray: + return _compute_separate_embeddings( + extractors, + mapping, + samples, + int(inference["outer_workers"]), + ) + + replacement, replacement_metrics = measure_call( + compute_embeddings, + psutil.Process(os.getpid()), + interval_seconds, + ) + baseline = np.asarray(prepared.embeddings, dtype=np.float32) + _validate_embedding_rows(baseline, len(mapping)) + labels, counter, clustering_seconds = _cluster_prepared( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + replacement, + ) + finalize_started = time.perf_counter() + diarization = engine.finalize(prepared, labels) + finalize_seconds = time.perf_counter() - finalize_started + operation_state.update( + { + "prepared": prepared, + "baseline_embeddings": baseline, + "embeddings": replacement, + "labels": labels, + "counter": counter, + "prepare_wall_seconds": prepare_wall, + "replacement_metrics": replacement_metrics, + "clustering_seconds": clustering_seconds, + "finalize_seconds": finalize_seconds, + } + ) + return diarization + + with capture_native_stderr() as native_log: + diarization, measured = measure_call( + operation, + psutil.Process(os.getpid()), + interval_seconds, + ) + native_log.flush() + native_log.seek(0) + prepare_metrics = parse_prepare_timings( + native_log.read().decode(errors="replace") + ) + validation_state: dict[str, Any] = {} + + def validate() -> None: + prepared = operation_state["prepared"] + validation_state["counter"] = _validate_cluster_repeatability( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + operation_state["labels"], + operation_state["counter"], + operation_state["embeddings"], + ) + baseline_labels, _, _ = _cluster_prepared( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + ) + baseline_diarization = engine.finalize(prepared, baseline_labels) + labels_equal = np.array_equal(operation_state["labels"], baseline_labels) + final_equal = _segments_signature(diarization) == _segments_signature( + baseline_diarization + ) + validation_state.update( + { + "labels_equal": labels_equal, + "final_equal": final_equal, + "drift": _embedding_drift( + operation_state["embeddings"], + operation_state["baseline_embeddings"], + ), + } + ) + + with capture_native_stderr(): + _, validation_measurement = measure_call( + validate, + psutil.Process(os.getpid()), + interval_seconds, + ) + replacement_metrics = operation_state["replacement_metrics"] + composed_wall = ( + operation_state["prepare_wall_seconds"] + - prepare_metrics["embedding_seconds"] + + replacement_metrics["wall_seconds"] + + operation_state["counter"]["seconds"] + + operation_state["clustering_seconds"] + + operation_state["finalize_seconds"] + ) + metrics = _composed_metrics(measured, composed_wall) + metrics.update( + { + "embedding_wall_seconds": replacement_metrics["wall_seconds"], + "embedding_peak_rss_bytes": replacement_metrics["peak_rss_bytes"], + **_validation_metrics(validation_measurement), + } + ) + stage_metrics = { + "segmentation_seconds": prepare_metrics["segmentation_seconds"], + "embedding_seconds": replacement_metrics["wall_seconds"], + "clustering_seconds": operation_state["clustering_seconds"], + "finalize_seconds": operation_state["finalize_seconds"], + "baseline_prepare_embedding_seconds": prepare_metrics["embedding_seconds"], + "engine_total_seconds": composed_wall, + "engine_audio_seconds": len(samples) / SAMPLE_RATE, + "engine_rtf": composed_wall / (len(samples) / SAMPLE_RATE), + } + prepared = operation_state["prepared"] + counters = dict(prepared.counters) + counters["embedding_run_calls"] = len(prepared.row_mapping) + rows_preserved = _prepared_rows_preserved(prepared) + validation_passed = bool( + rows_preserved + and validation_state["labels_equal"] + and validation_state["final_equal"] + ) + prepared_data = { + "counters": counters, + "counter": validation_state["counter"], + "row_mapping_count": len(prepared.row_mapping), + "row_mapping": tuple(prepared.row_mapping), + "row_labels": operation_state["labels"], + "embeddings": operation_state["embeddings"], + "validation_passed": validation_passed, + "inference": { + "mode": "separate-session", + "session_count": session_count, + "outer_workers": int(inference["outer_workers"]), + "intra_op_threads": int(inference["intra_op_threads"]), + "inter_op_threads": int(inference["inter_op_threads"]), + "rows_preserved": rows_preserved, + "finite_embeddings": True, + "labels_equal_to_baseline": validation_state["labels_equal"], + "final_equal_to_baseline": validation_state["final_equal"], + **validation_state["drift"], + }, + } + return diarization, metrics, stage_metrics, prepared_data + + +def _make_titanet_session(model_path: Path, inference: Mapping[str, Any]) -> Any: + import onnxruntime as ort + + options = ort.SessionOptions() + options.intra_op_num_threads = int(inference["intra_op_threads"]) + options.inter_op_num_threads = int(inference["inter_op_threads"]) + options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + return ort.InferenceSession( + str(model_path), + sess_options=options, + providers=["CPUExecutionProvider"], + ) + + +def _make_separate_extractors( + sherpa_onnx: Any, + cell: Mapping[str, Any], + artifacts: Mapping[str, Mapping[str, Any]], + provider_config_path: Path, +) -> list[Any]: + inference = cell["inference"] + config = sherpa_onnx.SpeakerEmbeddingExtractorConfig( + model=str(artifacts[cell["embedding_artifact_id"]]["path"]), + num_threads=int(inference["intra_op_threads"]), + provider=f"cpu:{provider_config_path}", + debug=False, + ) + return [ + sherpa_onnx.SpeakerEmbeddingExtractor(config) + for _ in range(int(inference["session_count"])) + ] + + +def _run_titanet_batch_recording( + engine: Any, + session: Any, + samples: np.ndarray, + cell: Mapping[str, Any], + recording: Mapping[str, Any], + interval_seconds: float, +) -> tuple[Any, dict[str, Any], dict[str, Any], Mapping[str, Any]]: + import psutil + import sherpa_onnx + + inference = cell["inference"] + selected_batch = int(inference["batch_size"]) + comparison_batch = 4 if selected_batch == 1 else 1 + operation_state: dict[str, Any] = {} + + def operation() -> Any: + prepare_started = time.perf_counter() + prepared = engine.prepare(samples, 1) + prepare_wall = time.perf_counter() - prepare_started + if prepared.is_terminal: + raise RuntimeError("titanet-batch не получил строки эмбеддингов") + mapping = tuple(prepared.row_mapping) + feature_started = time.perf_counter() + features = _prepare_titanet_features(mapping, samples) + feature_seconds = time.perf_counter() - feature_started + embeddings, inference_metrics = measure_call( + lambda: _infer_titanet_batches( + session, features, selected_batch + ), + psutil.Process(os.getpid()), + interval_seconds, + ) + labels, counter, clustering_seconds = _cluster_prepared( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + embeddings, + ) + finalize_started = time.perf_counter() + diarization = engine.finalize(prepared, labels) + finalize_seconds = time.perf_counter() - finalize_started + baseline = np.asarray(prepared.embeddings, dtype=np.float32) + _validate_embedding_rows(baseline, len(mapping)) + operation_state.update( + { + "prepared": prepared, + "prepare_wall_seconds": prepare_wall, + "feature_seconds": feature_seconds, + "features": features, + "baseline_embeddings": baseline, + "embeddings": embeddings, + "inference_metrics": inference_metrics, + "labels": labels, + "counter": counter, + "clustering_seconds": clustering_seconds, + "finalize_seconds": finalize_seconds, + } + ) + return diarization + + with capture_native_stderr() as native_log: + diarization, measured = measure_call( + operation, + psutil.Process(os.getpid()), + interval_seconds, + ) + native_log.flush() + native_log.seek(0) + prepare_metrics = parse_prepare_timings( + native_log.read().decode(errors="replace") + ) + validation_state: dict[str, Any] = {} + + def validate() -> None: + prepared = operation_state["prepared"] + selected_counter = _validate_cluster_repeatability( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + operation_state["labels"], + operation_state["counter"], + operation_state["embeddings"], + ) + comparison_embeddings, comparison_metrics = measure_call( + lambda: _infer_titanet_batches( + session, + operation_state["features"], + comparison_batch, + ), + psutil.Process(os.getpid()), + interval_seconds, + ) + comparison_labels, comparison_counter, _ = _cluster_prepared( + sherpa_onnx, + prepared, + cell, + int(recording["expected_speakers"]), + comparison_embeddings, + ) + comparison_diarization = engine.finalize(prepared, comparison_labels) + batches = { + selected_batch: { + "embeddings": operation_state["embeddings"], + "labels": operation_state["labels"], + "counter": selected_counter, + "diarization": diarization, + }, + comparison_batch: { + "embeddings": comparison_embeddings, + "labels": comparison_labels, + "counter": comparison_counter, + "diarization": comparison_diarization, + }, + } + labels_equal = np.array_equal(batches[1]["labels"], batches[4]["labels"]) + final_equal = _segments_signature( + batches[1]["diarization"] + ) == _segments_signature(batches[4]["diarization"]) + n_equal = ( + batches[1]["counter"]["num_clusters"] + == batches[4]["counter"]["num_clusters"] + ) + validation_state.update( + { + "selected_counter": selected_counter, + "comparison_metrics": comparison_metrics, + "batches": batches, + "labels_equal": labels_equal, + "final_equal": final_equal, + "n_equal": n_equal, + "drift": _embedding_drift( + batches[4]["embeddings"], batches[1]["embeddings"] + ), + "baseline_drift": _embedding_drift( + batches[1]["embeddings"], + operation_state["baseline_embeddings"], + ), + } + ) + + with capture_native_stderr(): + _, validation_measurement = measure_call( + validate, + psutil.Process(os.getpid()), + interval_seconds, + ) + selected_metrics = operation_state["inference_metrics"] + composed_wall = ( + operation_state["prepare_wall_seconds"] + - prepare_metrics["embedding_seconds"] + + operation_state["feature_seconds"] + + selected_metrics["wall_seconds"] + + operation_state["counter"]["seconds"] + + operation_state["clustering_seconds"] + + operation_state["finalize_seconds"] + ) + metrics = _composed_metrics(measured, composed_wall) + metrics.update( + { + "embedding_wall_seconds": selected_metrics["wall_seconds"], + "embedding_peak_rss_bytes": selected_metrics["peak_rss_bytes"], + **_validation_metrics(validation_measurement), + } + ) + stage_metrics = { + "segmentation_seconds": prepare_metrics["segmentation_seconds"], + "embedding_seconds": ( + operation_state["feature_seconds"] + selected_metrics["wall_seconds"] + ), + "clustering_seconds": operation_state["clustering_seconds"], + "finalize_seconds": operation_state["finalize_seconds"], + "baseline_prepare_embedding_seconds": prepare_metrics["embedding_seconds"], + "engine_total_seconds": composed_wall, + "engine_audio_seconds": len(samples) / SAMPLE_RATE, + "engine_rtf": composed_wall / (len(samples) / SAMPLE_RATE), + } + prepared = operation_state["prepared"] + counters = dict(prepared.counters) + counters["embedding_run_calls"] = math.ceil( + len(prepared.row_mapping) / selected_batch + ) + rows_preserved = _prepared_rows_preserved(prepared) + validation_passed = bool( + rows_preserved + and validation_state["n_equal"] + and validation_state["labels_equal"] + and validation_state["final_equal"] + ) + prepared_data = { + "counters": counters, + "counter": validation_state["selected_counter"], + "row_mapping_count": len(prepared.row_mapping), + "row_mapping": tuple(prepared.row_mapping), + "row_labels": operation_state["labels"], + "embeddings": operation_state["embeddings"], + "validation_passed": validation_passed, + "inference": { + "mode": "titanet-batch", + "batch_size": selected_batch, + "session_count": int(inference["session_count"]), + "intra_op_threads": int(inference["intra_op_threads"]), + "inter_op_threads": int(inference["inter_op_threads"]), + "variable_frame_lengths": sorted( + {len(item) for item in operation_state["features"]} + ), + "rows_preserved": rows_preserved, + "finite_embeddings": True, + "batch_1_n": validation_state["batches"][1]["counter"]["num_clusters"], + "batch_4_n": validation_state["batches"][4]["counter"]["num_clusters"], + "n_equal": validation_state["n_equal"], + "labels_equal": validation_state["labels_equal"], + "final_equal": validation_state["final_equal"], + **validation_state["drift"], + "baseline_batch_1_max_abs_drift": validation_state["baseline_drift"][ + "max_abs_drift" + ], + "baseline_batch_1_min_cosine": validation_state["baseline_drift"][ + "min_cosine" + ], + }, + } + return diarization, metrics, stage_metrics, prepared_data + + +def _run_public_cell(request: Mapping[str, Any]) -> dict[str, Any]: + import psutil + import sherpa_onnx + + manifest = request["manifest"] + cell = request["cell"] + patched = hasattr(sherpa_onnx.OfflineSpeakerDiarization, "prepare") + needs_patch = ( + cell["counter"] != "threshold" or cell["inference"]["mode"] != "sequential" + ) + if needs_patch and not patched: + raise RuntimeError("Ячейке нужен экспериментальный Prepare/Finalize") + mode = cell["inference"]["mode"] + if mode in {"separate-session", "titanet-batch"}: + _validate_special_inference( + cell, int(manifest["machine"]["physical_cores"]) + ) + + artifacts = {item["id"]: item for item in manifest["artifacts"]} + recordings = {item["id"]: item for item in manifest["recordings"]} + provider_config_path = ( + _write_provider_config(cell, Path(request["work_dir"])) if patched else None + ) + config = make_config(cell, artifacts, provider_config_path) + if not config.validate(): + raise RuntimeError("Конфигурация диаризатора недействительна") + engine = sherpa_onnx.OfflineSpeakerDiarization(config) + if engine.sample_rate != SAMPLE_RATE: + raise RuntimeError("Частота дискретизации модели не совпала") + special_runtime = None + if mode == "separate-session": + if provider_config_path is None: + raise RuntimeError("separate-session требует provider config") + special_runtime = _make_separate_extractors( + sherpa_onnx, + cell, + artifacts, + provider_config_path, + ) + elif mode == "titanet-batch": + special_runtime = _make_titanet_session( + Path(artifacts[cell["embedding_artifact_id"]]["path"]), + cell["inference"], + ) + + results = [] + for recording_id in cell["recording_ids"]: + recording = recordings[recording_id] + samples = _decode_clip(recording) + interval_seconds = float(manifest["rss_sample_interval_ms"]) / 1000.0 + prepared_data = None + if mode == "separate-session": + diarization, metrics, stage_metrics, prepared_data = ( + _run_separate_session_recording( + engine, + special_runtime, + samples, + cell, + recording, + interval_seconds, + ) + ) + progress_total = _optional_work_counter( + prepared_data["counters"], "embedding_jobs" + ) + progress_values_seen = _optional_work_counter( + prepared_data["counters"], "embedding_run_calls" + ) + elif mode == "titanet-batch": + diarization, metrics, stage_metrics, prepared_data = ( + _run_titanet_batch_recording( + engine, + special_runtime, + samples, + cell, + recording, + interval_seconds, + ) + ) + progress_total = _optional_work_counter( + prepared_data["counters"], "embedding_jobs" + ) + progress_values_seen = _optional_work_counter( + prepared_data["counters"], "embedding_run_calls" + ) + elif needs_patch: + diarization, metrics, stage_metrics, prepared_data = ( + _run_prepared_recording( + engine, + samples, + cell, + recording, + interval_seconds, + ) + ) + progress_total = _optional_work_counter( + prepared_data["counters"], "embedding_jobs" + ) + progress_values_seen = _optional_work_counter( + prepared_data["counters"], "embedding_run_calls" + ) + else: + progress, progress_state = _make_progress_callback() + with capture_native_stderr() as native_log: + diarization, metrics = measure_call( + lambda samples=samples, progress=progress: engine.process( + samples, progress + ), + psutil.Process(os.getpid()), + interval_seconds, + ) + native_log.flush() + native_log.seek(0) + stage_metrics = parse_stage_timings( + native_log.read().decode(errors="replace") + ) + progress_total = int(progress_state["total"]) + progress_values_seen = len(progress_state["seen"]) + segments = [ + { + "speaker": int(item.speaker), + "start": float(item.start), + "end": float(item.end), + } + for item in diarization.sort_by_start_time() + ] + intervals = [ + SpeakerInterval( + start=item["start"], end=item["end"], cluster=item["speaker"] + ) + for item in segments + ] + words = _load_words( + Path(recording["asr_words"]["path"]), float(recording["duration"]) + ) + transcript = build_speaker_transcript( + words, intervals, float(recording["duration"]) + ) + reference_turns = _read_reference_turns(recording) + word_rows = [] + for word in words: + assigned = _assign_cluster(word, intervals) + word_rows.append( + {"start": word.start, "end": word.end, "speaker": assigned} + ) + shift = int(float(cell["window_shift_ratio"]) * PYANNOTE_WINDOW_SAMPLES) + recording_result = { + "recording_id": recording_id, + "expected_speakers": recording["expected_speakers"], + "cluster_count": transcript.cluster_count, + "unassigned_word_count": transcript.unassigned_word_count, + "small_cluster_count": len(transcript.small_clusters), + "small_cluster_seconds": sum( + item.duration for item in transcript.small_clusters + ), + "mapped_speaker_purity": _mapped_speaker_purity(segments, reference_turns), + "text_input_sha256": _text_hash(words), + "text_output_sha256": _transcript_text_hash(transcript), + "text_equal": _text_hash(words) == _transcript_text_hash(transcript), + "window_shift_samples": shift, + "segmentation_windows": segmentation_window_count(len(samples), shift), + "embedding_jobs": progress_total, + "progress_values_seen": progress_values_seen, + "metrics": {**metrics, **stage_metrics}, + "segments": segments, + "word_assignments": word_rows, + **residual_cluster_metrics( + segments, word_rows, int(recording["expected_speakers"]) + ), + } + if prepared_data is not None: + matrix_dir = Path(request["work_dir"]) / "prepared" + matrix_dir.mkdir(parents=True, exist_ok=True) + matrix_path = matrix_dir / f"{cell['cell_id']}-{recording_id}.npz" + np.savez_compressed( + matrix_path, + embeddings=prepared_data["embeddings"], + row_labels=prepared_data["row_labels"], + row_mapping=np.asarray( + [item[:2] for item in prepared_data["row_mapping"]], + dtype=np.int32, + ), + sample_intervals_json=json.dumps( + [item[2] for item in prepared_data["row_mapping"]] + ), + ) + recording_result["prepared"] = { + "sha256": hashlib.sha256(matrix_path.read_bytes()).hexdigest(), + "rows": int(prepared_data["embeddings"].shape[0]), + "columns": int(prepared_data["embeddings"].shape[1]), + "row_mapping_count": prepared_data["row_mapping_count"], + "counters": prepared_data["counters"], + "counter": prepared_data["counter"], + "validation_passed": prepared_data.get("validation_passed", True), + } + if "inference" in prepared_data: + recording_result["prepared"]["inference"] = prepared_data["inference"] + results.append(recording_result) + + return { + "cell_name": cell["name"], + "recordings": results, + "aggregate": { + "wall_seconds": sum(item["metrics"]["wall_seconds"] for item in results), + "peak_rss_bytes": max( + item["metrics"]["peak_rss_bytes"] for item in results + ), + "quality_passed": all( + item["cluster_count"] == item["expected_speakers"] + and item["text_equal"] + and ( + "prepared" not in item + or ( + item["prepared"]["counters"].get("embedding_rows_dropped") + == 0 + and item["prepared"]["validation_passed"] + and item["prepared"]["counter"].get( + "repeatable_parameters", False + ) + and item["prepared"]["counter"].get( + "repeatable_labels", + item["prepared"]["counter"].get("status") == "terminal", + ) + ) + ) + for item in results + ), + }, + } + + +def run_cell(request: Mapping[str, Any]) -> dict[str, Any]: + """Выбирает публичный или патченный путь одной ячейки.""" + return _run_public_cell(request) diff --git a/tests/test_benchmark_experiment.py b/tests/test_benchmark_experiment.py new file mode 100644 index 0000000..0a02f34 --- /dev/null +++ b/tests/test_benchmark_experiment.py @@ -0,0 +1,923 @@ +import copy +import hashlib +import json +import os +import zipfile +from pathlib import Path + +import pytest + +from local_transcriber import benchmark_experiment as experiment + + +def _write(path: Path, content: bytes) -> dict[str, object]: + path.write_bytes(content) + return { + "path": str(path), + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } + + +def _artifact(tmp_path: Path, artifact_id: str, content: bytes) -> dict[str, object]: + return { + "id": artifact_id, + "kind": "onnx-model", + **_write(tmp_path / f"{artifact_id}.onnx", content), + "source_url": f"https://example.invalid/{artifact_id}", + "revision": "v1", + "license": "Apache-2.0", + } + + +def _cell(name: str, position: int, role: str) -> dict[str, object]: + return { + "name": name, + "stage": "intel", + "phase": "window", + "recording_ids": ["data-test", "t2-bdma", "yantar"], + "window_shift_ratio": 0.1 if role == "baseline" else 0.2, + "segmentation_artifact_id": "pyannote-fp32", + "embedding_artifact_id": "wespeaker-fp32", + "counter": "threshold", + "clustering": {"mode": "threshold", "threshold": 0.89, "num_clusters": None}, + "inference": { + "mode": "sequential", + "outer_workers": 1, + "session_count": 1, + "intra_op_threads": 4, + "inter_op_threads": 1, + "batch_size": 1, + }, + "repetition": 1, + "schedule_position": position, + "pair_id": "window-ab", + "pair_role": role, + "enabled": True, + "skip_reason": None, + "build_sha256": "9" * 64, + } + + +def _manifest(tmp_path: Path) -> dict[str, object]: + artifacts = [ + _artifact(tmp_path, "pyannote-fp32", b"segmentation"), + _artifact(tmp_path, "wespeaker-fp32", b"wespeaker"), + _artifact(tmp_path, "wespeaker-qdq", b"qdq"), + ] + recordings = [] + for index, (recording_id, speakers) in enumerate( + (("data-test", 3), ("t2-bdma", 2), ("yantar", 2)) + ): + media = _write(tmp_path / f"{recording_id}.wav", recording_id.encode()) + words = _write( + tmp_path / f"{recording_id}.words.json", + json.dumps([{"start": 0.0, "end": 0.5, "text": "слово"}]).encode(), + ) + reference = None + if index: + side = _write(tmp_path / f"{recording_id}.md", b"reference") + reference = { + "path": side["path"], + "sha256": side["sha256"], + "format": "hypescribe_markdown", + } + recordings.append( + { + "id": recording_id, + "role": "control", + "path": media["path"], + "sha256": media["sha256"], + "start": 0.0, + "duration": 300.0, + "expected_speakers": speakers, + "reference": reference, + "asr_words": { + "path": words["path"], + "sha256": words["sha256"], + "format": "local_transcriber_words_v1", + }, + } + ) + calibration = _write(tmp_path / "calibration.wav", b"separate calibration") + return { + "schema_version": 3, + "experiment": { + "id": "cpu-diarization-intel", + "stage": "intel", + "source_commit": "1" * 64, + "handoff_commit": None, + "sherpa_patch_sha256": "2" * 64, + "python_version": "3.13.0", + "dependencies": { + "sherpa-onnx": "1.13.6", + "sherpa-onnx-core": "1.13.6", + "onnxruntime": "1.28.0", + "numpy": "2.4.3", + "onnx": "1.20.0", + "kaldi-native-fbank": "1.22.3", + "psutil": "7.0.0", + }, + }, + "machine": { + "id": "intel-i7-6820hq", + "sku": "Intel Core i7-6820HQ", + "physical_cores": 4, + "logical_cores": os.cpu_count(), + "ram_bytes": 32 * 1024**3, + "os": "Linux test", + "cpu_flags": ["avx", "avx2"], + "power": { + "supply": "ac", + "profile": "balanced", + "governor": "powersave", + "turbo": True, + "power_limits": "platform-default", + "temperature_celsius": 55.0, + "throttling": False, + "swap_total_bytes": 8 * 1024**3, + "swap_used_bytes": 0, + }, + }, + "artifacts": artifacts, + "recordings": recordings, + "calibration": [ + { + "id": "calibration-1", + "path": calibration["path"], + "sha256": calibration["sha256"], + "source_id": "calibration-source-1", + "source_sha256": calibration["sha256"], + "source_start": 10.0, + "source_duration": 30.0, + "derived": False, + } + ], + "qdq": { + "enabled": True, + "source_ids": ["calibration-1"], + "preprocessing": {"sample_rate": 16000}, + "reader": {"name": "speaker-features-v1"}, + "quant_pre_process": {"skip_optimization": False}, + "quantize_static": { + "format": "QDQ", + "activation": "QInt8", + "weight": "QInt8", + }, + "excluded_nodes": [], + "tool_versions": {"onnxruntime": "1.28.0"}, + "graph_artifact_id": "wespeaker-qdq", + "skip_reason": None, + }, + "cells": [_cell("W0", 1, "baseline"), _cell("W2", 2, "candidate")], + "handoff": { + "outcome": "pending", + "source_commit": None, + "candidate_recipes": [], + "public_report_sha256": None, + "capsule_id": None, + "capsule_sha256": None, + }, + "rss_sample_interval_ms": 10, + } + + +def _environment() -> dict[str, object]: + return { + "packages": { + "sherpa-onnx": "1.13.6", + "sherpa-onnx-core": "1.13.6", + "onnxruntime": "1.28.0", + "numpy": "2.4.3", + "onnx": "1.20.0", + "kaldi-native-fbank": "1.22.3", + "psutil": "7.0.0", + }, + "python": "3.13.0", + "platform": "Linux test", + "logical_cores": os.cpu_count(), + "physical_cores": 4, + "cpu_sku": "Intel Core i7-6820HQ", + "cpu_flags": ["avx", "avx2"], + "ram_bytes": 32 * 1024**3, + "swap_total_bytes": 8 * 1024**3, + "swap_used_bytes": 0, + "build_sha256": "9" * 64, + } + + +def _guard(**changes) -> dict[str, object]: + result = { + "supply": "ac", + "profile": "balanced", + "governor": "powersave", + "turbo": True, + "power_limits": {"package": "platform-default"}, + "temperature_celsius": 55.0, + "throttling": False, + "throttle_count": 10, + "swap_used_bytes": 0, + "swap_sin_bytes": 100, + "swap_sout_bytes": 200, + } + result.update(changes) + return result + + +def test_manifest_identity_is_path_independent_and_covers_schedule(tmp_path): + manifest = _manifest(tmp_path) + experiment.validate_manifest(manifest) + first_id = experiment.make_cell_id(manifest, manifest["cells"][0]) + moved = copy.deepcopy(manifest) + for artifact in moved["artifacts"]: + artifact["path"] = "/private/moved/" + Path(artifact["path"]).name + for recording in moved["recordings"]: + recording["path"] = "/private/moved/recording" + recording["asr_words"]["path"] = "/private/moved/words" + if recording["reference"]: + recording["reference"]["path"] = "/private/moved/reference" + moved["calibration"][0]["path"] = "/private/moved/calibration" + + assert experiment.make_cell_id(moved, moved["cells"][0]) == first_id + + moved["cells"][0]["schedule_position"] = 3 + assert experiment.make_cell_id(moved, moved["cells"][0]) != first_id + assert experiment.make_experiment_id(moved) != experiment.make_experiment_id( + manifest + ) + + finalized = copy.deepcopy(manifest) + finalized["handoff"]["outcome"] = "stop-before-ryzen" + assert experiment.make_experiment_id(finalized) == experiment.make_experiment_id( + manifest + ) + + +def test_cell_identity_covers_sidecars_and_qdq_recipe(tmp_path): + manifest = _manifest(tmp_path) + cell = manifest["cells"][0] + cell["embedding_artifact_id"] = "wespeaker-qdq" + first_id = experiment.make_cell_id(manifest, cell) + + changed_sidecar = copy.deepcopy(manifest) + changed_sidecar["recordings"][0]["asr_words"]["sha256"] = "7" * 64 + assert experiment.make_cell_id(changed_sidecar, changed_sidecar["cells"][0]) != first_id + + changed_recipe = copy.deepcopy(manifest) + changed_recipe["qdq"]["quantize_static"]["weight"] = "QInt4" + assert experiment.make_cell_id(changed_recipe, changed_recipe["cells"][0]) != first_id + + +def test_identity_and_runtime_validation_cover_exact_versions_and_build(tmp_path): + manifest = _manifest(tmp_path) + first_id = experiment.make_cell_id(manifest, manifest["cells"][0]) + + changed_python = copy.deepcopy(manifest) + changed_python["experiment"]["python_version"] = "3.13.1" + assert ( + experiment.make_cell_id(changed_python, changed_python["cells"][0]) + != first_id + ) + + changed_numpy = copy.deepcopy(manifest) + changed_numpy["experiment"]["dependencies"]["numpy"] = "2.4.4" + assert ( + experiment.make_cell_id(changed_numpy, changed_numpy["cells"][0]) + != first_id + ) + with pytest.raises(ValueError, match="numpy==2.4.4"): + experiment.validate_environment(changed_numpy, _environment()) + + changed_build = copy.deepcopy(_environment()) + changed_build["build_sha256"] = "8" * 64 + with pytest.raises(ValueError, match="runtime-сборки"): + experiment.validate_environment(manifest, changed_build) + + +def test_manifest_rejects_stage_boundary_thread_budget_and_non_alternating_ab(tmp_path): + manifest = _manifest(tmp_path) + manifest["cells"][0]["stage"] = "ryzen" + with pytest.raises(ValueError, match="граница"): + experiment.validate_manifest(manifest, verify_files=False) + + manifest = _manifest(tmp_path) + manifest["cells"][0]["inference"]["outer_workers"] = 5 + with pytest.raises(ValueError, match="бюджет"): + experiment.validate_manifest(manifest, verify_files=False) + + manifest = _manifest(tmp_path) + manifest["cells"][1]["pair_role"] = "baseline" + with pytest.raises(ValueError, match="baseline/candidate"): + experiment.validate_manifest(manifest, verify_files=False) + + manifest = _manifest(tmp_path) + second_pair = [] + for source, name, position in ( + (manifest["cells"][0], "W0-repeat", 3), + (manifest["cells"][1], "W2-repeat", 4), + ): + cell = copy.deepcopy(source) + cell.update(name=name, schedule_position=position, pair_id="window-ab-2") + second_pair.append(cell) + manifest["cells"].extend(second_pair) + with pytest.raises(ValueError, match="начальная роль"): + experiment.validate_manifest(manifest, verify_files=False) + + +def test_ab_pair_requires_exactly_two_adjacent_schedule_positions(tmp_path): + manifest = _manifest(tmp_path) + manifest["cells"][1]["schedule_position"] = 3 + with pytest.raises(ValueError, match="соседними"): + experiment.validate_manifest(manifest, verify_files=False) + + manifest = _manifest(tmp_path) + for source, name, position in ( + (manifest["cells"][0], "W0-repeat", 3), + (manifest["cells"][1], "W2-repeat", 4), + ): + cell = copy.deepcopy(source) + cell.update(name=name, schedule_position=position) + manifest["cells"].append(cell) + with pytest.raises(ValueError, match="ровно одна пара"): + experiment.validate_manifest(manifest, verify_files=False) + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda manifest: manifest["cells"][0]["inference"].update(outer_workers=1.5), "целое"), + (lambda manifest: manifest["cells"][0].update(window_shift_ratio=0.3), "сетку"), + (lambda manifest: manifest["cells"][0]["clustering"].update(threshold=None), "число"), + ( + lambda manifest: manifest["cells"][0].update( + recording_ids=["data-test", "data-test"] + ), + "запись", + ), + ], +) +def test_manifest_rejects_fractional_or_ambiguous_cell_fields( + tmp_path, mutate, message +): + manifest = _manifest(tmp_path) + mutate(manifest) + + with pytest.raises((TypeError, ValueError), match=message): + experiment.validate_manifest(manifest, verify_files=False) + + +def test_manifest_rejects_enabled_qdq_without_graph(tmp_path): + manifest = _manifest(tmp_path) + manifest["qdq"]["graph_artifact_id"] = None + + with pytest.raises(ValueError, match="нужен граф"): + experiment.validate_manifest(manifest, verify_files=False) + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ({"supply": "battery"}, "от сети"), + ({"profile": "performance"}, "balanced"), + ({"throttling": True}, "нестабильно"), + ], +) +def test_manifest_rejects_unstable_power_contract(tmp_path, changes, message): + manifest = _manifest(tmp_path) + manifest["machine"]["power"].update(changes) + + with pytest.raises(ValueError, match=message): + experiment.validate_manifest(manifest, verify_files=False) + + +def test_resolve_manifest_paths_rejects_symlink_escape(tmp_path): + manifest = _manifest(tmp_path) + capsule = tmp_path / "capsule" + capsule.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (capsule / "escape").symlink_to(outside, target_is_directory=True) + manifest["artifacts"][0]["path"] = "escape/model.onnx" + + with pytest.raises(ValueError, match="выходит за корень"): + experiment.resolve_manifest_paths(manifest, capsule) + + +def test_raw_manifest_requires_capsule_relative_paths(tmp_path): + manifest = _manifest(tmp_path) + + with pytest.raises(ValueError, match="относительным"): + experiment.validate_manifest( + manifest, + verify_files=False, + require_relative_paths=True, + ) + + +def test_private_locations_inside_repository_must_be_git_ignored(): + repository_root = Path(experiment.__file__).parents[2] + + with pytest.raises(ValueError, match="не игнорируется Git"): + experiment._require_private_location( + repository_root / "docs" / "private-manifest.json", + "manifest", + ) + + experiment._require_private_location( + repository_root / "tmp" / "private-manifest.json", + "manifest", + ) + + +def test_qdq_rejects_control_origin_and_skips_missing_material_safely(tmp_path): + manifest = _manifest(tmp_path) + manifest["calibration"][0]["source_sha256"] = manifest["recordings"][0]["sha256"] + with pytest.raises(ValueError, match="контрольная запись"): + experiment.validate_manifest(manifest) + + manifest = _manifest(tmp_path) + Path(manifest["calibration"][0]["path"]).unlink() + warnings = experiment.validate_manifest(manifest) + plan = experiment.safe_plan(manifest, warnings, _environment()) + + assert plan["qdq_available"] is False + serialized = json.dumps(plan, ensure_ascii=False) + assert str(tmp_path) not in serialized + assert "separate calibration" not in serialized + + +@pytest.mark.parametrize( + "cluster_sizes, dimensions, seed, noise, expected", + [((3, 17), 4, 35, 0.1, 2), ((2, 3, 15), 3, 1, 0.3, 3)], +) +def test_nme_and_eigengap_count_block_fixtures( + cluster_sizes, dimensions, seed, noise, expected +): + embeddings = experiment.synthetic_block_embeddings( + cluster_sizes, dimensions=dimensions, seed=seed, noise=noise + ) + + assert experiment.estimate_eigengap(embeddings)["num_clusters"] == expected + assert experiment.estimate_nme(embeddings)["num_clusters"] == expected + + +def test_counter_selection_handles_mixed_model_outcomes(): + results = [] + for model_id, nme_counts, eigengap_counts, nme_time, eigengap_time in ( + ("campplus", [3, 2, 2], [3, 2, 2], 12.0, 11.5), + ("titanet", [3, 2, 2], [3, 3, 2], 8.0, 3.0), + ("broken", [3, 3, 2], [3, 2, 3], 1.0, 1.0), + ): + for counter, counts, wall in ( + ("nme", nme_counts, nme_time), + ("eigengap", eigengap_counts, eigengap_time), + ): + for num_clusters in counts: + results.append( + { + "embedding_id": model_id, + "counter": counter, + "num_clusters": num_clusters, + "wall_seconds": wall / 3, + } + ) + + assert experiment.select_counter(results) == { + "broken": None, + "campplus": "eigengap", + "titanet": "nme", + } + + +def test_window_selection_and_recipe_assembly_cover_step_only_and_deduplication(): + results = [ + { + "window_shift_ratio": 0.1, + "wall_seconds": 100.0, + "quality_passed": True, + "work_counters_explained": True, + }, + { + "window_shift_ratio": 0.15, + "wall_seconds": 88.0, + "quality_passed": True, + "work_counters_explained": True, + }, + { + "window_shift_ratio": 0.2, + "wall_seconds": 85.0, + "quality_passed": True, + "work_counters_explained": True, + }, + ] + assert experiment.select_working_shift(results) == 0.15 + assert experiment.assemble_candidate_recipes(0.15, []) == [ + { + "id": "C0", + "window_shift_ratio": 0.15, + "embedding": "wespeaker-fp32", + "inference": "sequential", + } + ] + + branches = [ + { + "quality_passed": True, + "memory_passed": True, + "diagnostic_approved": True, + "speedup": 0.2, + "recipe": {"embedding": "wespeaker-qdq", "inference": "shared-session"}, + "isa_stack": True, + "stack_order": 1, + }, + { + "quality_passed": True, + "memory_passed": True, + "diagnostic_approved": True, + "speedup": 0.15, + "recipe": {"embedding": "wespeaker-qdq", "inference": "shared-session"}, + "isa_stack": False, + }, + ] + recipes = experiment.assemble_candidate_recipes(0.15, branches) + assert len(recipes) <= 3 + assert len( + { + json.dumps({k: v for k, v in item.items() if k != "id"}, sort_keys=True) + for item in recipes + } + ) == len(recipes) + + unapproved = copy.deepcopy(branches) + unapproved[0]["diagnostic_approved"] = False + unapproved[0]["speedup"] = 0.9 + assert all( + item.get("embedding") != "wespeaker-qdq" + for item in experiment.assemble_candidate_recipes(0.15, unapproved[:1]) + ) + + +def test_schedule_preserves_raw_repeats_and_resumes_only_success(tmp_path): + manifest = _manifest(tmp_path) + output_path = tmp_path / "results.json" + work_dir = tmp_path / "work" + calls = [] + + def runner(request): + calls.append(request["cell"]["name"]) + return {"raw": request["cell"]["repetition"]} + + first = experiment.run_schedule( + manifest, + output_path, + work_dir, + runner, + environment=_environment(), + guard_reader=_guard, + ) + resumed_environment = _environment() + resumed_environment["swap_used_bytes"] = 4096 + second = experiment.run_schedule( + manifest, + output_path, + work_dir, + runner, + environment=resumed_environment, + guard_reader=_guard, + ) + + assert calls == ["W0", "W2"] + assert len(first["cells"]) == 2 + assert len(second["cells"]) == 2 + assert all(item["result"] == {"raw": 1} for item in second["cells"]) + assert all(item["guard"]["attempts"][0]["valid"] for item in second["cells"]) + + +def test_runtime_guard_retries_once_and_persists_transition_evidence(tmp_path): + manifest = _manifest(tmp_path) + manifest["cells"][1]["enabled"] = False + snapshots = iter( + [ + _guard(), + _guard(swap_sin_bytes=101), + _guard(temperature_celsius=57.0), + _guard(temperature_celsius=58.0), + ] + ) + calls = [] + + output = experiment.run_schedule( + manifest, + tmp_path / "results.json", + tmp_path / "work", + lambda request: calls.append(request["cell"]["name"]) or {"raw": 1}, + environment=_environment(), + guard_reader=lambda: next(snapshots), + ) + + assert calls == ["W0", "W0"] + attempts = output["cells"][0]["guard"]["attempts"] + assert [item["valid"] for item in attempts] == [False, True] + assert attempts[0]["reasons"] == ["swap_sin_bytes-changed"] + assert attempts[0]["before"]["temperature_celsius"] == 55.0 + assert attempts[1]["after"]["temperature_celsius"] == 58.0 + + +def test_runtime_guard_rejects_second_invalid_transition(tmp_path): + manifest = _manifest(tmp_path) + manifest["cells"][1]["enabled"] = False + snapshots = iter( + [ + _guard(), + _guard(throttle_count=11), + _guard(), + _guard(swap_sout_bytes=201), + ] + ) + output_path = tmp_path / "results.json" + + with pytest.raises(RuntimeError, match="W0"): + experiment.run_schedule( + manifest, + output_path, + tmp_path / "work", + lambda _request: {"raw": 1}, + environment=_environment(), + guard_reader=lambda: next(snapshots), + ) + + persisted = json.loads(output_path.read_text(encoding="utf-8"))["cells"][0] + assert persisted["status"] == "failed" + assert persisted["error_type"] == "RuntimeGuardError" + assert len(persisted["guard"]["attempts"]) == 2 + + +@pytest.mark.parametrize( + ("after", "reason"), + [ + (_guard(governor="performance"), "governor-mismatch"), + (_guard(turbo=False), "turbo-mismatch"), + (_guard(throttling=True), "throttling"), + ( + _guard(power_limits={"package": "changed"}), + "power-limits-changed", + ), + (_guard(swap_sout_bytes=201), "swap_sout_bytes-changed"), + ], +) +def test_runtime_guard_checks_stable_machine_conditions(tmp_path, after, reason): + reasons = experiment.validate_runtime_guard_transition( + _manifest(tmp_path), _guard(), after + ) + + assert reason in reasons + + +def test_failed_cell_is_persisted_without_private_exception_text(tmp_path): + manifest = _manifest(tmp_path) + output_path = tmp_path / "results.json" + + def runner(_request): + raise RuntimeError(f"private path: {tmp_path / 'secret.wav'}") + + with pytest.raises(RuntimeError, match="W0"): + experiment.run_schedule( + manifest, + output_path, + tmp_path / "work", + runner, + environment=_environment(), + guard_reader=_guard, + ) + + serialized = output_path.read_text(encoding="utf-8") + assert str(tmp_path) not in serialized + assert "secret.wav" not in serialized + assert "RuntimeError" in serialized + + +def test_residual_cluster_metrics_keep_time_and_word_units_separate(): + intervals = [ + {"speaker": 0, "start": 0.0, "end": 8.0}, + {"speaker": 1, "start": 8.0, "end": 10.0}, + {"speaker": 2, "start": 10.0, "end": 11.0}, + ] + words = [ + {"speaker": 0}, + {"speaker": 1}, + {"speaker": 2}, + {"speaker": 2}, + {"speaker": None}, + ] + + metrics = experiment.residual_cluster_metrics(intervals, words, 2) + + assert metrics["residual_speech_time_share"] == pytest.approx(1 / 11) + assert metrics["residual_assigned_word_share"] == pytest.approx(1 / 2) + + +def test_capsule_verification_checks_hashes_commits_and_safe_paths(tmp_path): + payload = b"private" + payload_hash = hashlib.sha256(payload).hexdigest() + manifest_path = tmp_path / "manifest.json" + manifest_bytes = b'{"schema_version":3}\n' + manifest_path.write_bytes(manifest_bytes) + capsule = { + "capsule_id": "opaque-41", + "source_commit": "3" * 64, + "public_report_sha256": "4" * 64, + "files": { + "manifest.json": hashlib.sha256(manifest_bytes).hexdigest(), + "data/payload.bin": payload_hash, + }, + } + archive_path = tmp_path / "capsule.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("capsule.json", json.dumps(capsule)) + archive.writestr("manifest.json", manifest_bytes) + archive.writestr("data/payload.bin", payload) + archive_hash = experiment.file_sha256(archive_path) + + result = experiment.verify_capsule( + archive_path, + archive_hash, + expected_manifest_path=manifest_path, + expected_source_commit="3" * 64, + expected_report_sha256="4" * 64, + expected_capsule_id="opaque-41", + ) + + assert result == { + "capsule_id": "opaque-41", + "source_commit": "3" * 64, + "public_report_sha256": "4" * 64, + "file_count": 2, + "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + } + + unrelated_manifest = tmp_path / "unrelated-manifest.json" + unrelated_manifest.write_bytes(b'{"schema_version":3,"other":true}\n') + with pytest.raises(ValueError, match="не совпал с manifest.json капсулы"): + experiment.verify_capsule( + archive_path, + archive_hash, + expected_manifest_path=unrelated_manifest, + ) + + unsafe_path = tmp_path / "unsafe.zip" + with zipfile.ZipFile(unsafe_path, "w") as archive: + archive.writestr("capsule.json", json.dumps(capsule)) + archive.writestr("manifest.json", manifest_bytes) + archive.writestr("../payload.bin", payload) + with pytest.raises(ValueError, match="небезопасный путь"): + experiment.verify_capsule( + unsafe_path, + experiment.file_sha256(unsafe_path), + expected_manifest_path=manifest_path, + ) + + +def test_capsule_binds_candidate_result_evidence_to_file_index(tmp_path): + result_bytes = b'{"aggregate":{"quality_passed":true}}' + result_hash = hashlib.sha256(result_bytes).hexdigest() + manifest_value = { + "handoff": { + "candidate_recipes": [ + { + "result_file": "results/candidate.json", + "result_sha256": result_hash, + } + ] + } + } + manifest_path = tmp_path / "manifest.json" + manifest_bytes = json.dumps(manifest_value, separators=(",", ":")).encode() + manifest_path.write_bytes(manifest_bytes) + capsule = { + "capsule_id": "opaque-41", + "source_commit": "3" * 64, + "public_report_sha256": "4" * 64, + "files": { + "manifest.json": hashlib.sha256(manifest_bytes).hexdigest(), + "results/candidate.json": result_hash, + }, + } + archive_path = tmp_path / "capsule.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("capsule.json", json.dumps(capsule)) + archive.writestr("manifest.json", manifest_bytes) + archive.writestr("results/candidate.json", result_bytes) + + experiment.verify_capsule( + archive_path, + experiment.file_sha256(archive_path), + expected_manifest_path=manifest_path, + ) + + broken_manifest_value = copy.deepcopy(manifest_value) + broken_manifest_value["handoff"]["candidate_recipes"][0]["result_sha256"] = ( + "5" * 64 + ) + broken_manifest_bytes = json.dumps( + broken_manifest_value, separators=(",", ":") + ).encode() + broken_manifest_path = tmp_path / "broken-manifest.json" + broken_manifest_path.write_bytes(broken_manifest_bytes) + capsule["files"]["manifest.json"] = hashlib.sha256( + broken_manifest_bytes + ).hexdigest() + broken_path = tmp_path / "broken-capsule.zip" + with zipfile.ZipFile(broken_path, "w") as archive: + archive.writestr("capsule.json", json.dumps(capsule)) + archive.writestr("manifest.json", broken_manifest_bytes) + archive.writestr("results/candidate.json", result_bytes) + with pytest.raises(ValueError, match="Результат кандидата"): + experiment.verify_capsule( + broken_path, + experiment.file_sha256(broken_path), + expected_manifest_path=broken_manifest_path, + ) + + +def test_handoff_enforces_candidate_limit_and_stop_contract(tmp_path): + manifest = _manifest(tmp_path) + manifest["handoff"]["outcome"] = "stop-before-ryzen" + manifest["handoff"]["candidate_recipes"] = ["C0"] + with pytest.raises(ValueError, match="ноль кандидатов"): + experiment.validate_manifest(manifest, verify_files=False) + + manifest = _manifest(tmp_path) + manifest["handoff"]["outcome"] = "handoff" + manifest["handoff"]["candidate_recipes"] = ["C0", "C1", "C2", "C3"] + with pytest.raises(ValueError, match="трех"): + experiment.validate_manifest(manifest, verify_files=False) + + +def test_finalize_handoff_binds_candidate_result_and_all_gates(tmp_path): + manifest = _manifest(tmp_path) + manifest["cells"][0]["phase"] = "combination" + cell_id = experiment.make_cell_id(manifest, manifest["cells"][0]) + payload = {"aggregate": {"quality_passed": True}, "raw": [1, 2, 3]} + output = { + "experiment_id": experiment.make_experiment_id(manifest), + "cells": [{"cell_id": cell_id, "status": "complete", "result": payload}], + } + + finalized = experiment.finalize_handoff( + manifest, + output, + ["W0"], + capsule_root=tmp_path / "capsule", + gate_evidence={ + "W0": {"memory_passed": True, "diagnostic_approved": True} + }, + public_report_sha256="4" * 64, + capsule_id="opaque-41", + ) + + candidate = finalized["handoff"]["candidate_recipes"][0] + assert candidate == { + "name": "W0", + "cell_id": cell_id, + "result_file": f"results/{cell_id}.json", + "result_sha256": experiment._digest_json(payload), + "mandatory_passed": True, + "memory_passed": True, + "diagnostic_approved": True, + } + result_file = tmp_path / "capsule" / candidate["result_file"] + assert experiment.file_sha256(result_file) == candidate["result_sha256"] + assert json.loads(result_file.read_text(encoding="utf-8")) == payload + experiment.validate_manifest(finalized, verify_files=False) + + finalized["handoff"]["capsule_sha256"] = "5" * 64 + experiment.validate_manifest(finalized, verify_files=False) + + +def test_finalize_handoff_rejects_missing_diagnostic_approval(tmp_path): + manifest = _manifest(tmp_path) + manifest["cells"][0]["phase"] = "combination" + cell_id = experiment.make_cell_id(manifest, manifest["cells"][0]) + output = { + "experiment_id": experiment.make_experiment_id(manifest), + "cells": [ + { + "cell_id": cell_id, + "status": "complete", + "result": {"aggregate": {"quality_passed": True}}, + } + ], + } + + with pytest.raises(ValueError, match="обязательные gates"): + experiment.finalize_handoff( + manifest, + output, + ["W0"], + capsule_root=tmp_path / "capsule", + gate_evidence={ + "W0": {"memory_passed": True, "diagnostic_approved": False} + }, + public_report_sha256="4" * 64, + capsule_id="opaque-41", + ) diff --git a/tests/test_diarization_calibration.py b/tests/test_diarization_calibration.py index f5f621e..ad7a85f 100644 --- a/tests/test_diarization_calibration.py +++ b/tests/test_diarization_calibration.py @@ -137,7 +137,7 @@ def _manifest(tmp_path): def _environment(): return { - "sherpa_onnx_version": "1.13.5", + "sherpa_onnx_version": "1.13.6", "onnxruntime_version": "1", "numpy_version": "2", "psutil_version": "7", @@ -408,7 +408,7 @@ def test_stage_timing_parser_accepts_native_lines(): "[I] OfflineSpeakerDiarization: clustering 0.100 s\n" "[I] OfflineSpeakerDiarization: total 3.100 s, audio 10.000 s, RTF 0.310" ) - stages = calibration.parse_stage_timings(log, "1.13.5") + stages = calibration.parse_stage_timings(log, "1.13.6") assert stages["engine_total_seconds"] == 3.1 assert stages["engine_audio_seconds"] == 10.0 assert stages["engine_rtf"] == 0.31 @@ -424,7 +424,7 @@ def test_stage_timing_parser_accepts_native_lines(): ) def test_stage_timing_parser_rejects_missing_duplicate_and_negative(log): with pytest.raises(ValueError): - calibration.parse_stage_timings(log, "1.13.5") + calibration.parse_stage_timings(log, "1.13.6") def test_stage_timing_parser_rejects_extra_malformed_record(): @@ -436,7 +436,7 @@ def test_stage_timing_parser_rejects_extra_malformed_record(): "OfflineSpeakerDiarization: total 3.100 s, audio 10.000 s, RTF 0.310" ) with pytest.raises(ValueError, match="повреждённую"): - calibration.parse_stage_timings(log, "1.13.5") + calibration.parse_stage_timings(log, "1.13.6") def test_measurement_uses_injected_cpu_clock_and_rss_once(): @@ -708,7 +708,7 @@ def test_execute_cell_rejects_text_mismatch_without_persisting_private_text( tmp_path, monkeypatch ): manifest = _manifest(tmp_path) - output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.5"}} + output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.6"}} spec = calibration.expand_automatic_cells(manifest, "e" * 64)[0] recording = manifest["recordings"][0] private_text = " СОВЕРШЕННО-СЕКРЕТНЫЙ-ТЕКСТ" @@ -758,7 +758,7 @@ def test_execute_cell_rejects_text_mismatch_without_persisting_private_text( def test_execute_cell_rejects_engine_audio_duration_mismatch(tmp_path, monkeypatch): manifest = _manifest(tmp_path) - output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.5"}} + output = {"environment": {"logical_cpus": 4, "sherpa_onnx_version": "1.13.6"}} spec = calibration.expand_automatic_cells(manifest, "e" * 64)[0] recording = manifest["recordings"][0] monkeypatch.setattr( diff --git a/tests/test_experimental_diarization.py b/tests/test_experimental_diarization.py new file mode 100644 index 0000000..e0481f6 --- /dev/null +++ b/tests/test_experimental_diarization.py @@ -0,0 +1,677 @@ +import io +import sys +from contextlib import contextmanager +from types import SimpleNamespace + +import numpy as np +import pytest + +from local_transcriber import experimental_diarization as runner + + +def _run_shared_cell(monkeypatch, tmp_path, *, counters=None, drop_output_text=False): + events = [] + active_interval = {"name": None} + embeddings = np.asarray([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32) + work_counters = ( + { + "embedding_jobs": 2, + "embedding_run_calls": 2, + "embedding_rows_dropped": 0, + } + if counters is None + else counters + ) + + class Diarization: + def sort_by_start_time(self): + return [ + SimpleNamespace(speaker=0, start=0.0, end=0.5), + SimpleNamespace(speaker=1, start=0.5, end=1.0), + ] + + class Engine: + sample_rate = runner.SAMPLE_RATE + + def __init__(self, config): + self.config = config + + def prepare(self, samples, workers): + events.append((active_interval["name"], f"prepare:{workers}")) + return SimpleNamespace( + is_terminal=False, + embeddings=embeddings.copy(), + row_mapping=((0, 0, [(0, 8_000)]), (1, 0, [(8_000, 16_000)])), + counters=dict(work_counters), + ) + + def finalize(self, prepared, labels): + events.append((active_interval["name"], "finalize")) + return Diarization() + + class Clustering: + def __init__(self, config): + self.config = config + + def __call__(self, values): + events.append((active_interval["name"], "cluster")) + return np.asarray([0, 1], dtype=np.int32) + + fake_sherpa = SimpleNamespace( + OfflineSpeakerDiarization=Engine, + FastClusteringConfig=lambda **values: SimpleNamespace(**values), + FastClustering=Clustering, + ) + monkeypatch.setitem(sys.modules, "sherpa_onnx", fake_sherpa) + monkeypatch.setattr( + runner, + "make_config", + lambda cell, artifacts, provider_config_path: SimpleNamespace( + validate=lambda: True + ), + ) + monkeypatch.setattr( + runner, + "_decode_clip", + lambda recording: np.zeros(runner.SAMPLE_RATE, dtype=np.float32), + ) + + def estimate(values): + events.append((active_interval["name"], "counter")) + return {"status": "complete", "num_clusters": 2, "p": 3} + + monkeypatch.setattr(runner, "estimate_nme", estimate) + measurement_names = iter(("published", "validation")) + + def measure(operation, process, interval_seconds): + name = next(measurement_names) + active_interval["name"] = name + try: + result = operation() + finally: + active_interval["name"] = None + suffix = 0 if name == "published" else 1 + return result, { + "wall_seconds": 10.0 + suffix, + "user_cpu_seconds": 7.0 + suffix, + "system_cpu_seconds": 1.0, + "total_cpu_seconds": 8.0 + suffix, + "average_cpu_cores": 0.8, + "peak_rss_bytes": 100 + suffix, + } + + monkeypatch.setattr(runner, "measure_call", measure) + + @contextmanager + def native_log(): + yield io.BytesIO() + + monkeypatch.setattr(runner, "capture_native_stderr", native_log) + monkeypatch.setattr( + runner, + "parse_prepare_timings", + lambda log: {"segmentation_seconds": 0.2, "embedding_seconds": 0.1}, + ) + if drop_output_text: + original_builder = runner.build_speaker_transcript + + def build_without_last_word(words, intervals, duration): + transcript = original_builder(words, intervals, duration) + transcript.turns = transcript.turns[:-1] + return transcript + + monkeypatch.setattr( + runner, "build_speaker_transcript", build_without_last_word + ) + + words_path = tmp_path / "words.json" + words_path.write_text( + '[{"start": 0.1, "end": 0.4, "text": "Привет"}, ' + '{"start": 0.6, "end": 0.9, "text": "мир"}]', + encoding="utf-8", + ) + cell = { + "name": "E1", + "cell_id": "shared-cell", + "counter": "nme", + "window_shift_ratio": 0.1, + "segmentation_artifact_id": "segmentation", + "embedding_artifact_id": "embedding", + "recording_ids": ["recording"], + "clustering": {"threshold": 0.89, "num_clusters": None}, + "inference": { + "mode": "shared-session", + "outer_workers": 2, + "session_count": 1, + "intra_op_threads": 2, + "inter_op_threads": 1, + "batch_size": 1, + }, + } + request = { + "manifest": { + "rss_sample_interval_ms": 10, + "artifacts": [], + "recordings": [ + { + "id": "recording", + "path": str(tmp_path / "recording.wav"), + "start": 0.0, + "duration": 1.0, + "expected_speakers": 2, + "asr_words": {"path": str(words_path)}, + "reference": None, + } + ], + }, + "cell": cell, + "work_dir": str(tmp_path), + } + return runner.run_cell(request), events + + +def test_run_cell_shared_session_separates_measurement_from_validation( + monkeypatch, tmp_path +): + result, events = _run_shared_cell(monkeypatch, tmp_path) + + recording = result["recordings"][0] + assert result["aggregate"]["quality_passed"] is True + assert recording["text_equal"] is True + assert recording["prepared"]["inference"] == { + "mode": "shared-session", + "rows_equal_to_sequential": True, + "finite_embeddings": True, + "labels_equal_to_sequential": True, + "final_equal_to_sequential": True, + "max_abs_drift": 0.0, + "min_cosine": 1.0, + } + assert recording["metrics"]["wall_seconds"] == 10.0 + assert recording["metrics"]["peak_rss_bytes"] == 100 + assert recording["metrics"]["validation_wall_seconds"] == 11.0 + assert recording["metrics"]["validation_peak_rss_bytes"] == 101 + assert events == [ + ("published", "prepare:2"), + ("published", "counter"), + ("published", "cluster"), + ("published", "finalize"), + ("validation", "counter"), + ("validation", "cluster"), + ("validation", "prepare:1"), + ("validation", "counter"), + ("validation", "cluster"), + ("validation", "finalize"), + ] + + +def test_run_cell_rejects_changed_output_text(monkeypatch, tmp_path): + result, _ = _run_shared_cell( + monkeypatch, tmp_path, drop_output_text=True + ) + + assert result["recordings"][0]["text_equal"] is False + assert result["aggregate"]["quality_passed"] is False + + +def test_run_cell_rejects_missing_prepared_work_counters(monkeypatch, tmp_path): + result, _ = _run_shared_cell(monkeypatch, tmp_path, counters={}) + + assert result["recordings"][0]["prepared"]["validation_passed"] is False + assert result["aggregate"]["quality_passed"] is False + + +def test_segmentation_window_count_includes_last_padded_window(): + assert runner.segmentation_window_count(160_000, 16_000) == 1 + assert runner.segmentation_window_count(160_001, 16_000) == 2 + assert runner.segmentation_window_count(176_000, 16_000) == 2 + + +def test_prepare_timing_parser_accepts_exactly_two_stages(): + log = ( + "OfflineSpeakerDiarization: segmentation 1.250 s\n" + "OfflineSpeakerDiarization: embedding 2.500 s" + ) + + assert runner.parse_prepare_timings(log) == { + "segmentation_seconds": 1.25, + "embedding_seconds": 2.5, + } + + with pytest.raises(ValueError, match="обе стадии"): + runner.parse_prepare_timings("OfflineSpeakerDiarization: embedding 2.5 s") + + +def test_known_counter_clusters_independent_embedding_copies(): + original = np.asarray([[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]], dtype=np.float32) + prepared = SimpleNamespace(embeddings=original, is_terminal=False) + + class FakeClustering: + calls = 0 + + def __init__(self, config): + self.config = config + + def __call__(self, features): + FakeClustering.calls += 1 + features[:] = -100 + return list(range(self.config.num_clusters)) + [0] + + fake_sherpa = SimpleNamespace( + FastClusteringConfig=lambda **values: SimpleNamespace(**values), + FastClustering=FakeClustering, + ) + cell = { + "counter": "known", + "clustering": {"threshold": 0.89}, + } + + labels, counter, _ = runner._cluster_prepared( + fake_sherpa, prepared, cell, expected_speakers=2 + ) + + assert labels.tolist() == [0, 1, 0] + assert counter["num_clusters"] == 2 + assert FakeClustering.calls == 1 + validated = runner._validate_cluster_repeatability( + fake_sherpa, + prepared, + cell, + expected_speakers=2, + labels=labels, + counter_result=counter, + ) + assert validated["repeatable_labels"] is True + assert FakeClustering.calls == 2 + assert np.array_equal(prepared.embeddings, original) + + +def _special_cell(mode, *, batch_size=1, sessions=1, outer=1, intra=4): + return { + "counter": "nme" if mode == "titanet-batch" else "known", + "clustering": {"threshold": 0.89}, + "embedding_artifact_id": "embedding", + "inference": { + "mode": mode, + "outer_workers": outer, + "session_count": sessions, + "intra_op_threads": intra, + "inter_op_threads": 1, + "batch_size": batch_size, + }, + } + + +def test_special_inference_enforces_exact_thread_budgets(): + runner._validate_special_inference( + _special_cell("separate-session", sessions=4, outer=4, intra=1), 4 + ) + runner._validate_special_inference( + _special_cell("titanet-batch", batch_size=4), 4 + ) + + with pytest.raises(ValueError, match="P worker"): + runner._validate_special_inference( + _special_cell("separate-session", sessions=2, outer=4, intra=1), 4 + ) + with pytest.raises(ValueError, match="бюджет физических ядер"): + runner._validate_special_inference( + _special_cell("separate-session", sessions=4, outer=4, intra=2), 4 + ) + with pytest.raises(ValueError, match="batch 1 или 4"): + runner._validate_special_inference( + _special_cell("titanet-batch", batch_size=2), 4 + ) + + +def test_separate_sessions_restore_row_order_without_loss(): + class Stream: + def __init__(self): + self.samples = [] + + def accept_waveform(self, sample_rate, samples): + assert sample_rate == runner.SAMPLE_RATE + self.samples.extend(samples) + + def input_finished(self): + pass + + class Extractor: + def create_stream(self): + return Stream() + + def is_ready(self, stream): + return bool(stream.samples) + + def compute(self, stream): + return [stream.samples[0], len(stream.samples)] + + samples = np.arange(30, dtype=np.float32) + mapping = [ + (0, 0, [(9, 12)]), + (0, 1, [(1, 3)]), + (1, 0, [(20, 24)]), + (1, 1, [(5, 6)]), + (2, 0, [(14, 19)]), + ] + + embeddings = runner._compute_separate_embeddings( + [Extractor(), Extractor()], mapping, samples, outer_workers=2 + ) + + assert embeddings.tolist() == [ + [9.0, 3.0], + [1.0, 2.0], + [20.0, 4.0], + [5.0, 1.0], + [14.0, 5.0], + ] + assert np.isfinite(embeddings).all() + + +def test_separate_runner_measures_validation_and_embedding_rss_separately( + monkeypatch, +): + embeddings = np.asarray([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32) + prepared = SimpleNamespace( + is_terminal=False, + embeddings=embeddings.copy(), + row_mapping=((0, 0, [(0, 5)]), (1, 0, [(5, 10)])), + counters={ + "embedding_jobs": 2, + "embedding_run_calls": 2, + "embedding_rows_dropped": 0, + }, + ) + events = [] + active_interval = {"name": None} + + class Clustering: + def __init__(self, config): + self.config = config + + def __call__(self, values): + events.append((active_interval["name"], "cluster")) + return np.asarray([0, 1], dtype=np.int32) + + fake_sherpa = SimpleNamespace( + FastClusteringConfig=lambda **values: SimpleNamespace(**values), + FastClustering=Clustering, + ) + monkeypatch.setitem(sys.modules, "sherpa_onnx", fake_sherpa) + + def estimate(values): + events.append((active_interval["name"], "counter")) + return {"status": "complete", "num_clusters": 2, "p": 3} + + monkeypatch.setattr(runner, "estimate_nme", estimate) + monkeypatch.setattr( + runner, + "_compute_separate_embeddings", + lambda extractors, mapping, samples, outer_workers: embeddings.copy(), + ) + measurement_names = iter(("published", "embedding", "validation")) + + def measure(operation, process, interval_seconds): + name = next(measurement_names) + previous = active_interval["name"] + active_interval["name"] = name + try: + result = operation() + finally: + active_interval["name"] = previous + values = { + "published": (10.0, 100), + "embedding": (3.0, 50), + "validation": (4.0, 200), + } + wall, peak = values[name] + return result, { + "wall_seconds": wall, + "user_cpu_seconds": wall / 2, + "system_cpu_seconds": 1.0, + "total_cpu_seconds": wall / 2 + 1.0, + "average_cpu_cores": 0.6, + "peak_rss_bytes": peak, + } + + monkeypatch.setattr(runner, "measure_call", measure) + + @contextmanager + def native_log(): + yield io.BytesIO() + + monkeypatch.setattr(runner, "capture_native_stderr", native_log) + monkeypatch.setattr( + runner, + "parse_prepare_timings", + lambda log: {"segmentation_seconds": 0.2, "embedding_seconds": 0.1}, + ) + + class Diarization: + def sort_by_start_time(self): + return [ + SimpleNamespace(speaker=0, start=0.0, end=0.5), + SimpleNamespace(speaker=1, start=0.5, end=1.0), + ] + + class Engine: + def prepare(self, samples, workers): + events.append((active_interval["name"], "prepare")) + return prepared + + def finalize(self, prepared_value, labels): + events.append((active_interval["name"], "finalize")) + return Diarization() + + cell = _special_cell("separate-session", sessions=2, outer=2, intra=1) + cell["counter"] = "nme" + _, metrics, _, data = runner._run_separate_session_recording( + Engine(), + [object(), object()], + np.zeros(10, dtype=np.float32), + cell, + {"expected_speakers": 2}, + 0.01, + ) + + assert metrics["peak_rss_bytes"] == 100 + assert metrics["embedding_peak_rss_bytes"] == 50 + assert metrics["validation_peak_rss_bytes"] == 200 + assert metrics["validation_wall_seconds"] == 4.0 + assert metrics["instrumented_pipeline_wall_seconds"] == 10.0 + assert "measured_validation_wall_seconds" not in metrics + assert data["validation_passed"] is True + assert events == [ + ("published", "prepare"), + ("published", "counter"), + ("published", "cluster"), + ("published", "finalize"), + ("validation", "counter"), + ("validation", "cluster"), + ("validation", "counter"), + ("validation", "cluster"), + ("validation", "finalize"), + ] + + +def test_titanet_batches_keep_variable_length_row_order(): + features = [ + np.full((3, 80), 10.0, dtype=np.float32), + np.full((5, 80), 20.0, dtype=np.float32), + np.full((2, 80), 30.0, dtype=np.float32), + np.full((4, 80), 40.0, dtype=np.float32), + np.full((6, 80), 50.0, dtype=np.float32), + ] + + class Session: + def __init__(self): + self.length_batches = [] + + def run(self, output_names, inputs): + assert output_names == ["embs"] + values = inputs["audio_signal"] + lengths = inputs["length"] + self.length_batches.append(lengths.tolist()) + assert values.shape == (len(lengths), 80, max(lengths)) + return [ + np.asarray( + [[values[index, 0, 0], length] for index, length in enumerate(lengths)], + dtype=np.float32, + ) + ] + + session = Session() + batch4 = runner._infer_titanet_batches(session, features, batch_size=4) + batch1 = runner._infer_titanet_batches(Session(), features, batch_size=1) + + assert session.length_batches == [[3, 5, 2, 4], [6]] + assert batch4.tolist() == batch1.tolist() == [ + [10.0, 3.0], + [20.0, 5.0], + [30.0, 2.0], + [40.0, 4.0], + [50.0, 6.0], + ] + + +def test_titanet_runner_compares_batch_1_and_4_before_finalize(monkeypatch): + embeddings = np.asarray( + [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]], dtype=np.float32 + ) + prepared = SimpleNamespace( + is_terminal=False, + embeddings=embeddings.copy(), + row_mapping=((0, 0, [(0, 4)]), (0, 1, [(4, 8)]), (1, 0, [(8, 12)])), + counters={"embedding_jobs": 3, "embedding_run_calls": 3, "embedding_rows_dropped": 0}, + ) + + events = [] + active_interval = {"name": None} + + class Clustering: + def __init__(self, config): + self.config = config + + def __call__(self, values): + events.append((active_interval["name"], "cluster")) + return np.asarray([0, 0, 1], dtype=np.int32) + + fake_sherpa = SimpleNamespace( + FastClusteringConfig=lambda **values: SimpleNamespace(**values), + FastClustering=Clustering, + ) + monkeypatch.setitem(sys.modules, "sherpa_onnx", fake_sherpa) + monkeypatch.setattr( + runner, + "estimate_nme", + lambda values: ( + events.append((active_interval["name"], "counter")) + or {"status": "complete", "num_clusters": 2, "p": 2} + ), + ) + monkeypatch.setattr( + runner, + "_prepare_titanet_features", + lambda mapping, samples: [ + np.ones((3, 80), dtype=np.float32), + np.ones((4, 80), dtype=np.float32), + np.ones((5, 80), dtype=np.float32), + ], + ) + seen_batches = [] + + def infer(session, features, batch_size): + seen_batches.append((active_interval["name"], batch_size)) + return embeddings.copy() + + monkeypatch.setattr(runner, "_infer_titanet_batches", infer) + measurement_names = iter( + ("published", "published-embedding", "validation", "validation-embedding") + ) + + def measure(operation, process, interval_seconds): + name = next(measurement_names) + previous = active_interval["name"] + active_interval["name"] = name + try: + result = operation() + finally: + active_interval["name"] = previous + return result, { + "wall_seconds": 1.0, + "user_cpu_seconds": 0.7, + "system_cpu_seconds": 0.1, + "total_cpu_seconds": 0.8, + "average_cpu_cores": 0.8, + "peak_rss_bytes": 100, + } + + monkeypatch.setattr(runner, "measure_call", measure) + + @contextmanager + def native_log(): + yield io.BytesIO() + + monkeypatch.setattr(runner, "capture_native_stderr", native_log) + monkeypatch.setattr( + runner, + "parse_prepare_timings", + lambda log: {"segmentation_seconds": 0.2, "embedding_seconds": 0.4}, + ) + + class Diarization: + def __init__(self, labels): + self.labels = labels + + def sort_by_start_time(self): + return [ + SimpleNamespace(speaker=int(label), start=index, end=index + 1) + for index, label in enumerate(self.labels) + ] + + class Engine: + def prepare(self, samples, workers): + assert workers == 1 + events.append((active_interval["name"], "prepare")) + return prepared + + def finalize(self, prepared_value, labels): + assert prepared_value is prepared + events.append((active_interval["name"], "finalize")) + return Diarization(labels) + + cell = _special_cell("titanet-batch", batch_size=4) + result, metrics, _, data = runner._run_titanet_batch_recording( + Engine(), + object(), + np.arange(20, dtype=np.float32), + cell, + {"expected_speakers": 2}, + 0.01, + ) + + assert seen_batches == [("published-embedding", 4), ("validation-embedding", 1)] + assert data["validation_passed"] is True + assert data["inference"]["n_equal"] is True + assert data["inference"]["labels_equal"] is True + assert data["inference"]["final_equal"] is True + assert data["counters"]["embedding_run_calls"] == 1 + assert metrics["measurement_kind"] == "composed-estimate" + assert metrics["wall_seconds_is_composed"] is True + assert metrics["instrumented_pipeline_wall_seconds"] == 1.0 + assert metrics["embedding_peak_rss_bytes"] == 100 + assert metrics["validation_wall_seconds"] == 1.0 + assert metrics["validation_peak_rss_bytes"] == 100 + assert events == [ + ("published", "prepare"), + ("published", "counter"), + ("published", "cluster"), + ("published", "finalize"), + ("validation", "counter"), + ("validation", "cluster"), + ("validation", "counter"), + ("validation", "cluster"), + ("validation", "finalize"), + ] + assert [item.speaker for item in result.sort_by_start_time()] == [0, 0, 1] diff --git a/uv.lock b/uv.lock index 59d5d46..1cafa88 100644 --- a/uv.lock +++ b/uv.lock @@ -1,13 +1,17 @@ version = 1 revision = 3 requires-python = ">=3.13" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version < '3.14'", +] [manifest] [[manifest.dependency-metadata]] name = "sherpa-onnx" -version = "1.13.5" -requires-dist = ["sherpa-onnx-core==1.13.5"] +version = "1.13.6" +requires-dist = ["sherpa-onnx-core==1.13.6"] [[package]] name = "annotated-doc" @@ -260,6 +264,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "kaldi-native-fbank" +version = "1.22.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/2c/84076b352107ce12d56f28c313f1aca1be332d953dd96aec7b84976e6d53/kaldi-native-fbank-1.22.3.tar.gz", hash = "sha256:387bf87225c6b83c93ae652eeaef1b4d531994b6e398e7a77189de340674f9af", size = 71013, upload-time = "2025-10-09T02:31:21.487Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/df/4110f685067946c8b2e59ed76cebdf51c979ae999d90f65208a9d1966cba/kaldi_native_fbank-1.22.3-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:78ca163686a4aa1693194d098aa79b517845d851aa6fd27d5b162c05e1012361", size = 249056, upload-time = "2025-10-09T02:28:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/d8/74/ef21aabdd2f32539735e2ed4d3ea072112d4e3d30dfc2d17695f6d9df072/kaldi_native_fbank-1.22.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3768ea99933aa25080cb820f93f7b612968633b9a4fa23bc8a7337e2137f3fbb", size = 229011, upload-time = "2025-10-09T02:31:50.593Z" }, + { url = "https://files.pythonhosted.org/packages/9a/72/adb11d27c545aca1db442da744ee430a6aae377a33574bfd2ec159dcf673/kaldi_native_fbank-1.22.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f74b85948328ab4b4c88522f98a59f83dd5295443b08483e945c7de2c35e5dcc", size = 299276, upload-time = "2025-10-09T02:30:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1e/496c7ae814b2a7f8f47d423dc33aae2cdfb1edf898e2faaf5c5b39b90363/kaldi_native_fbank-1.22.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3f9c6551ff5b6ae785dd15f819c3b2b7432d77bfb79ea8806748e2c7d900b5d", size = 322714, upload-time = "2025-10-09T02:30:32.698Z" }, + { url = "https://files.pythonhosted.org/packages/75/47/3fcb52e0ef081efa40b4ca5c04f207509d31157f33f3ac314578d93794f9/kaldi_native_fbank-1.22.3-cp313-cp313-win32.whl", hash = "sha256:a63d5bd6b5bd5f7c0e0af886c12c3f686fbc62347f6b886fed2694ab2f0dbd14", size = 273293, upload-time = "2025-10-09T02:30:13.979Z" }, + { url = "https://files.pythonhosted.org/packages/63/48/20bfa3f8d88605e2ec2c274c343dec1f112077e687440d64d3caa4b9136c/kaldi_native_fbank-1.22.3-cp313-cp313-win_amd64.whl", hash = "sha256:4fb769337c7d482166ada8ba041003e4a9de3a778dc970b6b5802a382e581724", size = 308032, upload-time = "2025-10-09T02:29:03.278Z" }, + { url = "https://files.pythonhosted.org/packages/b9/7e/d47f64d5332b2527e6b65490888d99793eb3280bca735d0b69348eaeb6a3/kaldi_native_fbank-1.22.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2efa8368cdd46a32c37a28c4baaa508b0a294ab1ca2aefddd3e97f62cfebc27b", size = 249216, upload-time = "2025-10-09T02:28:22.008Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/f98f72ba5a90a39675e82f2175dc5ec99a85892a88b9ccdd25f2dc916c82/kaldi_native_fbank-1.22.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8f6086073ec658a23d22f8657b3ee8c6ba69d65be57324a7284209ac7424b5ac", size = 229289, upload-time = "2025-10-09T02:31:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/d6/4b/1f3f17a7b601124df88112a1d1fcb543c8d908d6674f752f7d3322991770/kaldi_native_fbank-1.22.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:41fb506fde155d97aeef95dd6ceccc38c2c5dd4401f9b8fded9bacaf1bafef36", size = 300037, upload-time = "2025-10-09T02:30:10.203Z" }, + { url = "https://files.pythonhosted.org/packages/2b/6a/374ec4e1cf13e672f5acd8272116c1885c2a7f84be491fc652415fc6e870/kaldi_native_fbank-1.22.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f1cc2b8eeec52a33868cf59bb95d40b335fa9cff7e15a6208e0e9b67b7fd7236", size = 322854, upload-time = "2025-10-09T02:31:26.003Z" }, + { url = "https://files.pythonhosted.org/packages/63/2a/edd85a2292d2af28af68214a3bdd029ab8ce2e6bc5aaac77255aa57ce964/kaldi_native_fbank-1.22.3-cp314-cp314-win32.whl", hash = "sha256:d6387ab52b56e2978524590e11b24cf03419d9e9361965bc8d6ff34ff9e867da", size = 279733, upload-time = "2025-10-09T02:29:41.855Z" }, +] + [[package]] name = "local-transcriber" version = "0.1.0" @@ -279,6 +302,9 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "kaldi-native-fbank" }, + { name = "numpy" }, + { name = "onnx" }, { name = "psutil" }, { name = "pytest" }, ] @@ -292,13 +318,16 @@ requires-dist = [ { name = "onnxruntime", specifier = ">=1.28,<2" }, { name = "openvino-genai", marker = "(platform_machine == 'AMD64' and sys_platform != 'darwin') or (platform_machine == 'x86_64' and sys_platform != 'darwin')", specifier = ">=2026.3.0.0,<2026.4" }, { name = "rich", specifier = ">=14.3.3,<15" }, - { name = "sherpa-onnx", specifier = "==1.13.5" }, + { name = "sherpa-onnx", specifier = "==1.13.6" }, { name = "socksio", specifier = ">=1.0.0,<2" }, { name = "typer", specifier = ">=0.24.1,<1" }, ] [package.metadata.requires-dev] dev = [ + { name = "kaldi-native-fbank", specifier = ">=1.22,<2" }, + { name = "numpy", specifier = "==2.4.3" }, + { name = "onnx", specifier = ">=1.18,<2" }, { name = "psutil", specifier = ">=7,<8" }, { name = "pytest" }, ] @@ -324,6 +353,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "ml-dtypes" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/72/307d7c4bd0600601c7133fba5cb78af7db968152951c1cd473abb1cda782/ml_dtypes-0.6.0.tar.gz", hash = "sha256:5e60251d32ced5598972e4d5e06a2f044341f9291402551a3f6f0ec44f9299b0", size = 3032327, upload-time = "2026-08-13T14:14:40.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/51/fd1582b8f5ed8a9e7be0e161a6ea0dff70cb280479a12178df0b3a72700e/ml_dtypes-0.6.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:084dfe51a7ad58b171f05115f8226ed4233a454a1611371947e806e76f0c638d", size = 565468, upload-time = "2026-08-13T14:14:08.5Z" }, + { url = "https://files.pythonhosted.org/packages/d2/22/20fd70ca6ed12446cb92d5b2a7745bd185f9d8b8cdeeadad976574398e6b/ml_dtypes-0.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28d676428b104bb9717b0928bc5c5129f2d6b51b6727587cc4289e7bf8713cb5", size = 360232, upload-time = "2026-08-13T14:14:09.873Z" }, + { url = "https://files.pythonhosted.org/packages/89/a5/da8ae6c6f1babe4b68e3e55d43d39b529e29774f10e0910671a6b8c86eb8/ml_dtypes-0.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26b1f1fa4f0435a2946859823f6e2bf06796f1e9f10f5a05b08a5e3c8f46ff69", size = 410169, upload-time = "2026-08-13T14:14:11.036Z" }, + { url = "https://files.pythonhosted.org/packages/e2/55/4561acefa00fa4bcbfb82ca6a48578b41f372cd7dd7cdd6eb4720abc2e5f/ml_dtypes-0.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:fb87f46b4f7ad7b5d3ad8f4b452b024bd4229d44c8ff934798c1fe656210387a", size = 439357, upload-time = "2026-08-13T14:14:12.172Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5d/6a01538e507ef0ed5e879985b13a92467bf8960696fb1131f8b8cadc60ff/ml_dtypes-0.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:57ed0d6b4ac5e7868361303a9c57fbcf63b768236ee14456f585dfcf260d0292", size = 552278, upload-time = "2026-08-13T14:14:13.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7a/97dc35667b7c9db33c5344c673cd27f87e34771875ea7100138726132ac9/ml_dtypes-0.6.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:84fa136b8602c8c39e3b6cb24918960cd6f36cade7a70376f56770729cd56510", size = 562551, upload-time = "2026-08-13T14:14:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/db/48/77f0ede10558d0d935da2e3276ed7e9c8cc2bad3463b9a0b66b03fc60be2/ml_dtypes-0.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:317be9967fb84b0ce4e80e6b1bf71213d21971621cf6f1e501a63602a95297bf", size = 360334, upload-time = "2026-08-13T14:14:16.079Z" }, + { url = "https://files.pythonhosted.org/packages/1c/b1/1831dd8c9b06c013085d31a2ac4f03392d43bd36bfc6ff591a08bcedc1cf/ml_dtypes-0.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f490c003369ce60e514a0c3b12374f05274c101fee1bead6740ec8a564032b0", size = 409966, upload-time = "2026-08-13T14:14:17.477Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ad/9c32c53f823dda3742df19a79c10bc198365937873ea125ba65747440c23/ml_dtypes-0.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:d574c2b28921dc72e869df248f1a278f6eee176a1f237c8642e1a71eb15f3977", size = 457224, upload-time = "2026-08-13T14:14:18.608Z" }, + { url = "https://files.pythonhosted.org/packages/41/3d/dd98205418a13353d41c52bf5326d8cbec515aace46174e23c6ea01c2978/ml_dtypes-0.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:f4adb4af61516510d786cf8c01851a66f6d3ddfa79e1144deaa5b40d8507231e", size = 568378, upload-time = "2026-08-13T14:14:19.843Z" }, + { url = "https://files.pythonhosted.org/packages/65/36/32e7beef3281fed74883451477ad976364323206dbfaa95e948ba788dac7/ml_dtypes-0.6.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:3e169214e0d80ff1c038e1b3017e33c23e43bdf948d42d31de8283111c7e2fa3", size = 590177, upload-time = "2026-08-13T14:14:20.971Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/99b3d9b3c984b3bd1e81d8244f1fa2f812e44060d853205b2df6271aa17c/ml_dtypes-0.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:573b11f3c327e17ef3826d266e676cf1149a1f3016f822a05f2306c55d8246bf", size = 363142, upload-time = "2026-08-13T14:14:22.463Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fb/8091c0aee7f2712de99c7fd4b1642382644dec6a4962effe4f5b9d16a973/ml_dtypes-0.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b76fa1d3f92967d58289ac47ab7458ede66e6f3527fff3e59142aee57d9307cd", size = 430645, upload-time = "2026-08-13T14:14:23.737Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6f/962d2c589513b5930d05b6eae5fbd22ad8bbcf26bb763449f3d8f912360f/ml_dtypes-0.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:3be9911d953f97cddded4b9961d7b650473b7e55806d20f6176f8356dfe7b38e", size = 465667, upload-time = "2026-08-13T14:14:25.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ca/bcb25e246edd19af5fa1cf6267040bd9977a7afca846e6cfd4a52078b44f/ml_dtypes-0.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:e74266ca8e97874a937b7646378c178025650a236584f7474d10d8086a6edea3", size = 572706, upload-time = "2026-08-13T14:14:26.296Z" }, + { url = "https://files.pythonhosted.org/packages/12/42/46cb442648e3c774d8cb25f2e1e41d496cdcc91fbe9c2a6f75c0b8df7af6/ml_dtypes-0.6.0-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:b1b503864fada3f74fabf8d9fee7b4c1cbe956301e6fdece975d5f77c2fce958", size = 562550, upload-time = "2026-08-13T14:14:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/07/56/844eff5af7a2d1a09d75df12c70225c3a6b6a771f95876b2bf5f7d10ad44/ml_dtypes-0.6.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c6ad60af4102789a5c09824004beade2f7f28cd1cd581ee5c170d9dc2fbb00e", size = 360332, upload-time = "2026-08-13T14:14:28.767Z" }, + { url = "https://files.pythonhosted.org/packages/b6/29/b7165a3a76364a5baa6aa4ee82a0adf73a3c014b8cd126120b62cc087992/ml_dtypes-0.6.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4f1b9329a251e4affe3bb58f4d3e2db22a714396fd7ffb40d0b5db423c24d17", size = 409964, upload-time = "2026-08-13T14:14:30.023Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2e/f61c54a0544b6a170ac1bb89bcf406af53fb2deffc5476b6d2d3df5ba13e/ml_dtypes-0.6.0-cp315-cp315-win_amd64.whl", hash = "sha256:488c99ab181a2f59d9ec3b12c5fa11ec904e92be2c4ba18cded54dd7501208fe", size = 457249, upload-time = "2026-08-13T14:14:31.213Z" }, + { url = "https://files.pythonhosted.org/packages/63/00/bee1bc9faa02a46e7a851019fd23f47ca1f906609edbec8b6ba5decc3cc3/ml_dtypes-0.6.0-cp315-cp315-win_arm64.whl", hash = "sha256:de9d14748dbf3968951436ef514a29c9d1fe438aa680d110134ee2f7a9f9df18", size = 568381, upload-time = "2026-08-13T14:14:32.548Z" }, + { url = "https://files.pythonhosted.org/packages/72/f7/9a5edede28f73185fd51d75030ef7f11d76997bab3a92427d986e54fe2eb/ml_dtypes-0.6.0-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:e25bb3b0ad1217b60626e4ed45b10ca170c41d99fbe44a12bebc1e07ec4aad55", size = 589877, upload-time = "2026-08-13T14:14:33.695Z" }, + { url = "https://files.pythonhosted.org/packages/fd/81/d5924a141b850b606eb027493c9c3ca3c665cca5163af3f5b6e5e3345503/ml_dtypes-0.6.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:31f1ce979d31a357e95aa81812f20412c8c954fa43c44ee3ead1e1c8a78575ef", size = 362788, upload-time = "2026-08-13T14:14:34.996Z" }, + { url = "https://files.pythonhosted.org/packages/59/8f/3298e3f334832bc28dd144af6b99cdc93502a8687e71922ea68b0a319929/ml_dtypes-0.6.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2d6149f3a57f405bcad5fb41e03218b8373936253f23e1ca84c0108abbc3392", size = 430823, upload-time = "2026-08-13T14:14:36.44Z" }, + { url = "https://files.pythonhosted.org/packages/93/d2/f2dbf118f42ce4c325a139c9236737f436b7f8e00cd18701c99ef2405e6f/ml_dtypes-0.6.0-cp315-cp315t-win_amd64.whl", hash = "sha256:ce7563e0b1a4482cbc1b4a6272145e54e4489e54fe7428f94908c3d87103abfa", size = 465119, upload-time = "2026-08-13T14:14:37.776Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ff/bda40387b5c5c64254595f4d81a12351770856acc5de4e6d43606a31f161/ml_dtypes-0.6.0-cp315-cp315t-win_arm64.whl", hash = "sha256:f6cb525101b6b903779188c1e9e9490c343b455ab822883e02cf01e5547338d2", size = 572666, upload-time = "2026-08-13T14:14:38.993Z" }, +] + [[package]] name = "numpy" version = "2.4.3" @@ -382,6 +447,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/3c/aa88abe01f3be3d1f8f787d1d33dc83e76fec05945f9a28fbb41cfb99cd5/nvidia_cublas_cu12-12.9.1.4-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:453611eb21a7c1f2c2156ed9f3a45b691deda0440ec550860290dc901af5b4c2", size = 581242350, upload-time = "2025-06-05T20:04:51.979Z" }, ] +[[package]] +name = "onnx" +version = "1.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/19/8ea73a64b368b75fe339771a20a02bc61ea1f551484c9e3d9d0bfbd0450f/onnx-1.22.0.tar.gz", hash = "sha256:ef40c0aaf0b643857ea9306fc7eddce17eaf9fb0407e4801f1fc5758443a38e0", size = 12024721, upload-time = "2026-06-15T12:50:05.354Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/6a/481561f1093834376ed493e4ca42a73e5be0d50031f2969c86593bdc7c96/onnx-1.22.0-cp312-abi3-macosx_12_0_universal2.whl", hash = "sha256:596fbf0490947533c1c1045ba860851dc9fb77471023dac9a71ba5b42ceab103", size = 20167081, upload-time = "2026-06-15T12:49:32.078Z" }, + { url = "https://files.pythonhosted.org/packages/84/55/b34fc2aa30aa54b4a775402d24c4082242c720283a274fe976ac8eb94480/onnx-1.22.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae5a563f281cd9d2845622cecf6c092a57e4ee1b138f66fdbbdd4200567a5e16", size = 18889249, upload-time = "2026-06-15T12:49:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/09/a6/bd32357e6cc1ecb473afd78193d7231724f284435d2db25696ecfaaa1503/onnx-1.22.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:955e02e1f6d385b53d52f9cd7b9cdf5caf417c300bcfe3c64c6d542be763845b", size = 19106514, upload-time = "2026-06-15T12:49:37.424Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9d/3af461ac6c714b8b369cb71499659932f4f12cfb066250b62f7567c3d530/onnx-1.22.0-cp312-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:82e9f27fc1223cb06d68a56bed6f9d3caf3d0dad1b61bce45006d529b15bd94c", size = 16966387, upload-time = "2026-06-15T12:49:40.918Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f0/68195b5e5a53e333faf2660f5352ee43738d0e42fc5216cc6b1871a9fbfb/onnx-1.22.0-cp312-abi3-win32.whl", hash = "sha256:cc8b66b312f8f03a53e268afb67180a2d97dd12cc79e2b61361c6c0073448016", size = 17081568, upload-time = "2026-06-15T12:49:43.398Z" }, + { url = "https://files.pythonhosted.org/packages/13/a8/734725bb703c5fabb687f79c79e51249475212b3eb37771ac4a4ac9b487f/onnx-1.22.0-cp312-abi3-win_amd64.whl", hash = "sha256:72ccebab3bac07215c204ce8848d42e78eaaa666badbf72d25cd359b9f269e3a", size = 17213290, upload-time = "2026-06-15T12:49:45.933Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/8ce48d8ae26a8761ad4e5dc771961b155c5c3c7c8540ec7f2f2d71b69af0/onnx-1.22.0-cp312-abi3-win_arm64.whl", hash = "sha256:f3c120dcdb70ad738f3c061b32798f408ea299eb69f84dd69ab4a6bf3c2ec01f", size = 17207030, upload-time = "2026-06-15T12:49:48.635Z" }, + { url = "https://files.pythonhosted.org/packages/f3/13/47323b97846387848efb1044ded11bb94b83526f3d1fbdb37c6480d4520f/onnx-1.22.0-cp314-cp314t-macosx_12_0_universal2.whl", hash = "sha256:19e45e4af88e3fe3261458d4b8cc461957ae2782a358a3560503569bf3b23b72", size = 20176465, upload-time = "2026-06-15T12:49:51.311Z" }, + { url = "https://files.pythonhosted.org/packages/13/0c/d3b8a7e7eee123938586c608bb9894b5723f2342b9450c0eec59fbec7099/onnx-1.22.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c21a0e59fd967a95b358e4a6e756d1f1eec2d304a83480f329f66e30d2bf0223", size = 18894028, upload-time = "2026-06-15T12:49:54.451Z" }, + { url = "https://files.pythonhosted.org/packages/b8/8a/da2a97ab46fe6e0cd9beb3ac14603a22f5be492f9ca347faf8233a07bb33/onnx-1.22.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2632406b8f523ef2e2873c363f90b20a3d88c0fbcfac757d3addffccf8f452c2", size = 19110420, upload-time = "2026-06-15T12:49:57.665Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a3/ce984063017518307ebfaa545782fc400e593dc2d7fdf4f23ce4be1ed197/onnx-1.22.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a3a39fc4643867aecb33417fdddb11e308ee79d2d4a584b9d50cc7aec2091b13", size = 17237547, upload-time = "2026-06-15T12:50:00.382Z" }, + { url = "https://files.pythonhosted.org/packages/00/50/257a880384a1dd502d543b0067945074d63cd17d0840e958355bc8197da8/onnx-1.22.0-cp314-cp314t-win_arm64.whl", hash = "sha256:8e268cdc0547e3949799ffd4a44451dc2b9080b57d0824a2db680b6ec65506f0", size = 17231391, upload-time = "2026-06-15T12:50:03.047Z" }, +] + [[package]] name = "onnx-asr" version = "0.12.0" @@ -516,6 +607,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/e7/14dc9366696dcb53a413449881743426ed289d687bcf3d5aee4726c32ebb/protobuf-7.34.0-py3-none-any.whl", hash = "sha256:e3b914dd77fa33fa06ab2baa97937746ab25695f389869afdf03e81f34e45dc7", size = 170716, upload-time = "2026-02-27T00:30:23.994Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -525,22 +644,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] -[[package]] -name = "psutil" -version = "7.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.169Z" }, - { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, - { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.977Z" }, - { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.795Z" }, - { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.343Z" }, - { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, - { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.515Z" }, -] - [[package]] name = "pytest" version = "9.0.2" @@ -626,47 +729,47 @@ wheels = [ [[package]] name = "sherpa-onnx" -version = "1.13.5" +version = "1.13.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sherpa-onnx-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6d/a1/8a7d8751bc71a0814f3b9332717909512a5ff919d07ed53e4baa861ef8c7/sherpa_onnx-1.13.5.tar.gz", hash = "sha256:14bebfe71365a2c678dd94cd08efa8e79df06318b17fe8e97b2e802a7881fd5a", size = 1037285, upload-time = "2026-08-11T08:17:31.817Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/12/ca1900b2ad6308f76b9f744367748ce81adab2fb70a5665acf0a6f3c34f0/sherpa_onnx-1.13.6.tar.gz", hash = "sha256:d55a0e769efd9b014c2d3aaa54754288dc39c7261e9334c575d54ba22214ca27", size = 1037965, upload-time = "2026-08-18T06:55:13.107Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/61/6f/b70ae4c7e367e3d2dca99efc65152d22ad5d942619add19172d24267c14b/sherpa_onnx-1.13.5-cp313-cp313-linux_armv7l.whl", hash = "sha256:15a02d9d74143f336156cb8b4da826bc65c11daa28bd5c9099e9397b3660e7df", size = 11999210, upload-time = "2026-08-11T09:26:07.356Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ec/d471d042cc85c505515e3fbaed55223f54718a9ac3a8f228a8e1b81dab6d/sherpa_onnx-1.13.5-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:371b0eb51caa12f5f3e9c327c440a4aee6f77c163c89666469844eaefb7d5351", size = 4439247, upload-time = "2026-08-11T08:30:33.777Z" }, - { url = "https://files.pythonhosted.org/packages/28/58/2f60eb29db5f41c01ad844be34ed1adfdecce9493b12cc20bbdc6b6ad5a8/sherpa_onnx-1.13.5-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:2bdfefa5d16896c90d1685bde43682818e70b95e134dccc95b91dc0819adde1c", size = 2340028, upload-time = "2026-08-11T07:18:47.753Z" }, - { url = "https://files.pythonhosted.org/packages/6e/71/25f067216edcde5e78cf0d9a7ac053a2b3b17bdf131acac69447daa23356/sherpa_onnx-1.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:05a3effd8ea5ee78735bc28909c0fce26d9f23bf61db663e6f75f04ec0247ba4", size = 2137522, upload-time = "2026-08-11T07:43:35.186Z" }, - { url = "https://files.pythonhosted.org/packages/5a/77/b36cd40906bef1b3a6593200587eea5c4e7c1e94d29c4fd88a876e1cd6ca/sherpa_onnx-1.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5a6cc5ac96043670faa0f5c0e56310315a4600cf7b764fee014e7dd75fda00f", size = 4167203, upload-time = "2026-08-11T07:48:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/9a/3b/817cdb5770f8beafd22a04a8eda0c8253a6d07a518a09ce982d6e4152e3c/sherpa_onnx-1.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb7c1f5ca5455b3ec008f3e09438bca6038d22b4fe5d64adb5c09bb6b84eeb82", size = 4392107, upload-time = "2026-08-11T08:03:49.714Z" }, - { url = "https://files.pythonhosted.org/packages/55/75/b8494e30923a18db144e0035851f0364c32e63d776b62f3d2667f717f499/sherpa_onnx-1.13.5-cp313-cp313-win32.whl", hash = "sha256:80d9ec3ce2fe8b566285ef92fa2e72b4fa0aee10f73290ac74219fe415806980", size = 1961338, upload-time = "2026-08-11T08:35:46.591Z" }, - { url = "https://files.pythonhosted.org/packages/5e/28/e8644784a897e4ba6cdc7a635d821107ce31567c90e1835a0c008ba2a322/sherpa_onnx-1.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:440174433b5d3f855a800d757e66812e998ec9715f5f42690afb27a6ff456586", size = 2279915, upload-time = "2026-08-11T08:51:50.171Z" }, - { url = "https://files.pythonhosted.org/packages/02/99/1aa369391f62a2753b5d97b0b9ecd99475410c51eada10f7079628a58c4f/sherpa_onnx-1.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:38de7f31754dfa7212beb4cc773ba05f78d028adb4c0e008920128388657f778", size = 2239783, upload-time = "2026-08-11T07:46:44.679Z" }, - { url = "https://files.pythonhosted.org/packages/21/a3/ac238819cca1955b6b80ae802671cc7e4d688c673fb98001c21111922045/sherpa_onnx-1.13.5-cp314-cp314-linux_armv7l.whl", hash = "sha256:a224466e750a435089311d4c763617a4d83478a2f958c311b473ea59f9cce3ae", size = 11997852, upload-time = "2026-08-11T08:54:11.47Z" }, - { url = "https://files.pythonhosted.org/packages/02/82/3f5b50f84b9a8571bdfe9939983a8e6dff75110961305315887bbc83f626/sherpa_onnx-1.13.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:1a4ea5fa45caf31093ff832ab4d198e097edbeeb2244a0d4f958e96bf888110a", size = 4442303, upload-time = "2026-08-11T07:36:28.234Z" }, - { url = "https://files.pythonhosted.org/packages/fe/90/6d3a1112bc4496b0b918d3a807036912f1f34fb4a4dc506f0cabf398c4a9/sherpa_onnx-1.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7582ca42ed2d80bacc984b46904cbdd058ec2b6149a8c8996d7263217a4cc963", size = 2340527, upload-time = "2026-08-11T07:54:35.482Z" }, - { url = "https://files.pythonhosted.org/packages/84/9d/0cb152e3fd1aa5787ade99c4863cd8cf80d800b879e8ff47f8f99412af78/sherpa_onnx-1.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97471fe025fc1d655a1df2f4ffb5d3fe843c269f2016d5e1cca4b0d168a49169", size = 2140247, upload-time = "2026-08-11T08:23:33.405Z" }, - { url = "https://files.pythonhosted.org/packages/e6/84/38531f16e2c2a0b077257b25c34c02a1c8f7f1f833d5bb61e2b2f510e260/sherpa_onnx-1.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2a51751f6c925c405ef6ebcb2f7c44794810da79d0f2743fe57ee3daa5bc4df5", size = 4172385, upload-time = "2026-08-11T08:00:34.952Z" }, - { url = "https://files.pythonhosted.org/packages/45/91/56a439945713a5cf49080a833fbd0953936fcd50c651c17afe64478c9922/sherpa_onnx-1.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:eb4d580d959e16c42dbd404cef0f5764ed1c024edf48d3edc43b5e2f1c96f069", size = 4393503, upload-time = "2026-08-11T07:38:11.97Z" }, - { url = "https://files.pythonhosted.org/packages/d5/27/cbdfa8bdcf618b28ec5cf383b073be851ff3178ea06d6f1a0fcb3c7532e8/sherpa_onnx-1.13.5-cp314-cp314-win32.whl", hash = "sha256:64cb492cf8a9751f632685c6f169fef7e5a2f449efb9923bc21b6e783a911dba", size = 2000807, upload-time = "2026-08-11T08:57:11.235Z" }, - { url = "https://files.pythonhosted.org/packages/d6/c2/162840897a528a528bc0f76cf3516d4e3ac1445db9089caf7733df756cfc/sherpa_onnx-1.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:001e1c3440fe335a589c9283a68105f379ffb63f2d1d7c2878e99a5d72eaf5af", size = 2345302, upload-time = "2026-08-11T08:11:46.752Z" }, - { url = "https://files.pythonhosted.org/packages/28/5e/9a9c63e1dcd646c5f21df6d16e6770fedd956f8a54241f3382343c6289af/sherpa_onnx-1.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:a7e97d8457a51725ffa3fcb44cb1d57bfdb780bc390a9460a65e05bf3d2851b4", size = 2303336, upload-time = "2026-08-11T07:33:25.917Z" }, + { url = "https://files.pythonhosted.org/packages/48/52/5ad50062b757b0d99b297594b106173bdcc50480f512dd5ae97cd0d2bc09/sherpa_onnx-1.13.6-cp313-cp313-linux_armv7l.whl", hash = "sha256:788aef27900e15d1bf6482c6f745a26f6ded9b41aaa1fddbacabf6a20366e523", size = 11999760, upload-time = "2026-08-18T08:47:38.139Z" }, + { url = "https://files.pythonhosted.org/packages/87/2a/31a94c922813471ef8284658d4dfeae0ceb1d94de08cd8c6728aa95f4e79/sherpa_onnx-1.13.6-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:4438843292ddcff7267e79c032b4711d3f54529434a5e225c820cc0da4c489a4", size = 4439567, upload-time = "2026-08-18T07:20:58.922Z" }, + { url = "https://files.pythonhosted.org/packages/8e/47/1d1e726b4f15bd66cbe0b73bf81334764d6025e05738d984ebdd96a78e0e/sherpa_onnx-1.13.6-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:0c357e37fbb958edd93841ba40723b2447d5470fc683329b8eadd32d454320c3", size = 2340457, upload-time = "2026-08-18T06:39:53.405Z" }, + { url = "https://files.pythonhosted.org/packages/fc/04/552c85d7f0b6ba1a1008da0568054d5c834c644f26057a24baf92dcdf876/sherpa_onnx-1.13.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdb0f9fcaae89f8acd1ee9ce5fe89e77be3aa240cdedab3f48aa7b19800d8b72", size = 2137534, upload-time = "2026-08-18T07:59:20.279Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f3/5d50f5e675d972c39d908f82b554e6e1734c2d165e6573b8b92e7b0f3aaa/sherpa_onnx-1.13.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:248e3d039dba42ffeb141e7cb914f297e23199fca94e0413259dcdf5fb281960", size = 4168068, upload-time = "2026-08-18T06:28:33.845Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3e/c3a60cf13301363210e7a0cc72a901ae6a56e313e1d912df0017e24e6e79/sherpa_onnx-1.13.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5212391170d5ac2fdd02109b313cc1120393ededc2a06dd9eed731a4ca24f068", size = 4391526, upload-time = "2026-08-18T06:41:25.459Z" }, + { url = "https://files.pythonhosted.org/packages/a5/fe/26d6a5aaae30f79e987ca55e482f6a92839e579e1eb0b6dc2987e1390b5d/sherpa_onnx-1.13.6-cp313-cp313-win32.whl", hash = "sha256:1743cf209361647430d83f44cd2ddd686f74a9a8442699427de4223ed1b17066", size = 1961677, upload-time = "2026-08-18T07:15:46.315Z" }, + { url = "https://files.pythonhosted.org/packages/cd/04/a5cd9dd2a6e5073ca5fb8df28a95ecb49df71a8b3ff822aa57f294dec985/sherpa_onnx-1.13.6-cp313-cp313-win_amd64.whl", hash = "sha256:3387c754668578612aaf7795f894b631faefb71fd7d7df20c2aa41aa2dbc59a2", size = 2280022, upload-time = "2026-08-18T07:52:46.014Z" }, + { url = "https://files.pythonhosted.org/packages/c3/a4/0e0194e561c6670315a0efc09ec80680ae43d196572974abba624ad9ac90/sherpa_onnx-1.13.6-cp313-cp313-win_arm64.whl", hash = "sha256:b52143c11ff064a02b80c231ce462eb8e258fc5a7b994c750ee8145ca79c114d", size = 2240745, upload-time = "2026-08-18T07:35:12.686Z" }, + { url = "https://files.pythonhosted.org/packages/41/a3/62a284cba29e3c9307f24794a65167371cca1390c3b690458f15fcbb75e3/sherpa_onnx-1.13.6-cp314-cp314-linux_armv7l.whl", hash = "sha256:a7c05186d279db4e1ef8949d798c88a05b524f95743966667b70f70ea18243e2", size = 11997843, upload-time = "2026-08-18T08:41:04.762Z" }, + { url = "https://files.pythonhosted.org/packages/2b/7a/fd4a1b1e1580cbb6623fbb06f1b05afac69a4d84867541f4e653a422d8d8/sherpa_onnx-1.13.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:3105aec59b691ba59c75148261c2d7a64614fbeb7470b8116c04e8ec01aa9514", size = 4442856, upload-time = "2026-08-18T08:02:22.368Z" }, + { url = "https://files.pythonhosted.org/packages/be/de/ba9022fb7d5b7e7818bb456d10220702468c8be0ecfe3d164347b165439b/sherpa_onnx-1.13.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:afd08d1337d14d8f27f70802096c237044b3fc9c922d8b40d7f8b12d84566a10", size = 2341119, upload-time = "2026-08-18T07:29:01.085Z" }, + { url = "https://files.pythonhosted.org/packages/b3/32/8889ae5e10eaf1c29d0e9f7ff63ad9b5168dd7304274666d23c4a48aa841/sherpa_onnx-1.13.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:89ca7cd437bf250be60d297ffdb58b77acc5c6beef2d90e84029a0cfaa8783fe", size = 2140675, upload-time = "2026-08-18T08:10:34.166Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a0/05b594030df8c52e9a89e1615e3b6d59f99db1c884a33f6d5d2782045650/sherpa_onnx-1.13.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:370cc39be2613f27640e9fd6e326556109e2202d11d1420c8ba1616fc4eb1b90", size = 4173222, upload-time = "2026-08-18T06:45:53.883Z" }, + { url = "https://files.pythonhosted.org/packages/bd/92/638cb914a924d9039270113c8ecdd419a82a56072141bea85d8a88433ade/sherpa_onnx-1.13.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:243e8dfb4749b4ded8893f9e6af90615a16c5df1ad7c8aa211f5207911cfe9b2", size = 4393166, upload-time = "2026-08-18T07:08:41.428Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/389b69bb78339d498ffee9f2ce40e8c27d55f46f45606aaafa23c9af586f/sherpa_onnx-1.13.6-cp314-cp314-win32.whl", hash = "sha256:9b0bffb23a428139983c4f42753b924097b0181a74257e2e529bd70f4fb49e9b", size = 2001391, upload-time = "2026-08-18T07:24:22.179Z" }, + { url = "https://files.pythonhosted.org/packages/20/7c/f7f4d89088114a8b861c8b5416f14c67ca3bdae652ea2e9499f438c17dc8/sherpa_onnx-1.13.6-cp314-cp314-win_amd64.whl", hash = "sha256:aa4321c92ec8b12f602664551c72cbe0c920f5002009dc9457d6e092639b0ae2", size = 2345347, upload-time = "2026-08-18T07:51:40.427Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0d/caf428b986911a9027e9e30d2735456557be3e7ea86ab70b8cd2b7fc7514/sherpa_onnx-1.13.6-cp314-cp314-win_arm64.whl", hash = "sha256:6b5861db46d126a6b97b05432e4122ba2b8505f17f2ee16c1b377c9f3251ba36", size = 2303436, upload-time = "2026-08-18T07:33:28.755Z" }, ] [[package]] name = "sherpa-onnx-core" -version = "1.13.5" +version = "1.13.6" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/fe/2a972bc05bc1c674275a97d16d387545c6efa811bf2a8dc1b6ac573f9fe1/sherpa_onnx_core-1.13.5-py3-none-macosx_10_15_universal2.whl", hash = "sha256:a5380f4f8ec609e1c68f2052e34a55862098fa0efa0bba0af33adbdc0231fc4c", size = 19850412, upload-time = "2026-08-11T07:15:17.913Z" }, - { url = "https://files.pythonhosted.org/packages/27/cd/5730372ce10ba08fda03a3e3dad4ec8bc2a62607530a8f32c44e63258c51/sherpa_onnx_core-1.13.5-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:6bb5c460cd432293a290c861fbd519e30ac290ca84b5acd1c4447c22aef03351", size = 10614601, upload-time = "2026-08-11T07:00:58.916Z" }, - { url = "https://files.pythonhosted.org/packages/91/54/b753e2dbd2b0f09aed2d2082f35c41606af9712457278ff99d13eaae92d3/sherpa_onnx_core-1.13.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:899e88916efd96ee1eabc512e9ceaf2fdb711b20b2512cf40358b623e90b5ded", size = 9282744, upload-time = "2026-08-11T07:09:47.547Z" }, - { url = "https://files.pythonhosted.org/packages/e6/58/505668cedd7036bcba305c1a1f5424a3340f4701297a628de83d61e8ed58/sherpa_onnx_core-1.13.5-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4cd751063a378a49f0c72eba5ba959fe375397f5baf93a53f3db64097d00e2aa", size = 13051193, upload-time = "2026-08-11T07:00:47.526Z" }, - { url = "https://files.pythonhosted.org/packages/8f/61/2cf3aee64c280c830ca4fa25584eb87f8acf04787152744a29d576323977/sherpa_onnx_core-1.13.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:485b20188ddf905ffbd7504ed456a05d0ff84d10c592a16c0adb7d69bb845f82", size = 10371666, upload-time = "2026-08-11T07:19:12.791Z" }, - { url = "https://files.pythonhosted.org/packages/24/08/c04236fdf7825c4149b8b28b24d5a26f9e3fdca879aa24b8f1c65e96745b/sherpa_onnx_core-1.13.5-py3-none-manylinux_2_35_armv7l.whl", hash = "sha256:12697ff5745e225157b3257fe638e8b6305b1860366b6387e7a586ee6de2b18b", size = 9920766, upload-time = "2026-08-11T08:48:07.866Z" }, - { url = "https://files.pythonhosted.org/packages/44/c0/6d19a5b4be1541e2bd17ec045d0c48a641055d290007826ddee3a081a905/sherpa_onnx_core-1.13.5-py3-none-win32.whl", hash = "sha256:6b2f19b7559ad278aa17af0b9f3917a90a37fef4501c04afb14e722f4848c9b7", size = 14534505, upload-time = "2026-08-11T08:22:22.047Z" }, - { url = "https://files.pythonhosted.org/packages/2a/67/61611b5b945d7ac860387283a352860df8ab0431e3cc0868c1d5291e9f86/sherpa_onnx_core-1.13.5-py3-none-win_amd64.whl", hash = "sha256:fb48f5b7b4596bb745e5dcbb04616d841712224cba652717a71a4551d8ee5b63", size = 16522273, upload-time = "2026-08-11T07:09:46.3Z" }, - { url = "https://files.pythonhosted.org/packages/28/b2/d7ee13aba3fc40316e2363a5ac0780be7b0112050f7e8821bcd00d6af024/sherpa_onnx_core-1.13.5-py3-none-win_arm64.whl", hash = "sha256:7e24d3ddbcb1e6447467b214ff1235dced21fa4107de5c55125fb3c95ff10501", size = 15683505, upload-time = "2026-08-11T07:11:03.608Z" }, + { url = "https://files.pythonhosted.org/packages/df/cd/633f4a3eb8841e9d849dd130d20498a9315be03d27f5050c2ca139a16226/sherpa_onnx_core-1.13.6-py3-none-macosx_10_15_universal2.whl", hash = "sha256:365539159406cd41ff5031e5db68d6e402a392a15b3a77b2e952f579c1945f3c", size = 19850670, upload-time = "2026-08-18T06:14:56.282Z" }, + { url = "https://files.pythonhosted.org/packages/2c/89/2ee5d5d79f3bea896d89caff95d641881b58ddd629d538801b15cef2747c/sherpa_onnx_core-1.13.6-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:f5a672a02d42d1c369ddd129f8be57bfe8b7e785140a13976ff7633e27c4ffdd", size = 10614866, upload-time = "2026-08-18T06:29:24.317Z" }, + { url = "https://files.pythonhosted.org/packages/06/ee/dbbc6718e263e4961106e057c187875e00485ca8030a6bca0d9682807d30/sherpa_onnx_core-1.13.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2a413121dd0ee2e53c45def3ea4e8f0e968a6ac74f8cfd288bed3ae2cc815c33", size = 9282956, upload-time = "2026-08-18T07:21:24.118Z" }, + { url = "https://files.pythonhosted.org/packages/58/ab/ab279cf4eb6d4aa4b5f703d4e8233365c3acee304b963a09d4cb71601324/sherpa_onnx_core-1.13.6-py3-none-manylinux2014_aarch64.whl", hash = "sha256:e247b203ca64ab7095864d3ef1a21159787e02a985a7df137652604377c7833e", size = 13051341, upload-time = "2026-08-18T06:14:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/fa/0a/6f64568ebb230b200c30c465eb002113944705fa36c734838ea46f2e1802/sherpa_onnx_core-1.13.6-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9c1260ae150ebd4f658854fd8d8ca8477828afa5bebf66c1b2f6132f1f704a21", size = 10371677, upload-time = "2026-08-18T06:15:23.693Z" }, + { url = "https://files.pythonhosted.org/packages/21/ba/170e3de104c091e95e49d8f547473d58f925ec55add3729b49e3498efce4/sherpa_onnx_core-1.13.6-py3-none-manylinux_2_35_armv7l.whl", hash = "sha256:21e3e0c90cb9d3117d8ea64b3c3e598e6b85098b35582fed6ebb742625d33fc6", size = 9920659, upload-time = "2026-08-18T07:43:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/5d/29/470348cf8a8e580f8db99f7f3a6f49bba4657ea0ad8de58f15c8e4be83f0/sherpa_onnx_core-1.13.6-py3-none-win32.whl", hash = "sha256:97103228dde9b772a53c200a5c1e0dc1821f09e3139e0a72431a95a28a4eb62a", size = 14534700, upload-time = "2026-08-18T06:31:06.45Z" }, + { url = "https://files.pythonhosted.org/packages/81/61/d32923b9f6fa039271760e1c642ffd496b8e2b3340e084bd9b0c7b568448/sherpa_onnx_core-1.13.6-py3-none-win_amd64.whl", hash = "sha256:9e7ac522b4f108412f6a1173c0e512c65e98d4714d9d9aa0002279d588d2e699", size = 16522526, upload-time = "2026-08-18T06:51:39.613Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/025e55fc07fde16a7abe19f0d1616469e8e4b56424323f0721f321a064ab/sherpa_onnx_core-1.13.6-py3-none-win_arm64.whl", hash = "sha256:36ce28bbcab720627e53befa1adc4573ffb966d9c9ce7d78e260cd8a5edb9103", size = 15685909, upload-time = "2026-08-18T06:37:19.099Z" }, ] [[package]] From 0ea4d56c9d35a2776e46057db43e05d21eab8bc1 Mon Sep 17 00:00:00 2001 From: Dmitry Dementiev Date: Mon, 31 Aug 2026 15:16:10 +0300 Subject: [PATCH 2/4] =?UTF-8?q?fix(diarization):=20=D1=83=D1=87=D1=82?= =?UTF-8?q?=D1=91=D0=BD=20=D1=81=D0=B8=D1=81=D1=82=D0=B5=D0=BC=D0=BD=D1=8B?= =?UTF-8?q?=D0=B9=20power=20profile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Зачем: - часть Intel-стендов не публикует ACPI platform_profile - balanced-режим на таком стенде доступен через power-profiles-daemon Что: - добавлен fallback на powerprofilesctl get - поведение закреплено тестом Проверка: - uv run pytest -q tests/test_benchmark_experiment.py - uv run ruff check для изменённых файлов --- src/local_transcriber/benchmark_experiment.py | 20 ++++++++++++++++++- tests/test_benchmark_experiment.py | 13 ++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/local_transcriber/benchmark_experiment.py b/src/local_transcriber/benchmark_experiment.py index 1e5008e..8c8ac91 100644 --- a/src/local_transcriber/benchmark_experiment.py +++ b/src/local_transcriber/benchmark_experiment.py @@ -931,6 +931,24 @@ def _runtime_supply() -> str | None: return None +def _runtime_profile() -> str | None: + profile = _read_optional_text(Path("/sys/firmware/acpi/platform_profile")) + if profile is not None: + return profile + try: + completed = subprocess.run( + ["powerprofilesctl", "get"], + capture_output=True, + text=True, + check=False, + timeout=5, + ) + except (OSError, subprocess.TimeoutExpired): + return None + value = completed.stdout.strip() + return value or None if completed.returncode == 0 else None + + def _runtime_governor() -> str | None: values = { value @@ -1006,7 +1024,7 @@ def runtime_guard_snapshot() -> dict[str, Any]: swap = psutil.swap_memory() return { "supply": _runtime_supply(), - "profile": _read_optional_text(Path("/sys/firmware/acpi/platform_profile")), + "profile": _runtime_profile(), "governor": _runtime_governor(), "turbo": _runtime_turbo(), "power_limits": _runtime_power_limits(), diff --git a/tests/test_benchmark_experiment.py b/tests/test_benchmark_experiment.py index 0a02f34..fb1ed26 100644 --- a/tests/test_benchmark_experiment.py +++ b/tests/test_benchmark_experiment.py @@ -593,6 +593,19 @@ def test_schedule_preserves_raw_repeats_and_resumes_only_success(tmp_path): assert all(item["guard"]["attempts"][0]["valid"] for item in second["cells"]) +def test_runtime_profile_falls_back_to_powerprofilesctl(monkeypatch): + monkeypatch.setattr(experiment, "_read_optional_text", lambda _path: None) + monkeypatch.setattr( + experiment.subprocess, + "run", + lambda *args, **kwargs: experiment.subprocess.CompletedProcess( + args[0], 0, stdout="balanced\n", stderr="" + ), + ) + + assert experiment._runtime_profile() == "balanced" + + def test_runtime_guard_retries_once_and_persists_transition_evidence(tmp_path): manifest = _manifest(tmp_path) manifest["cells"][1]["enabled"] = False From 6345cb6449d906767b6ed33e695f2ca4eeaf3008 Mon Sep 17 00:00:00 2001 From: Dmitry Dementiev Date: Mon, 31 Aug 2026 15:55:40 +0300 Subject: [PATCH 3/4] =?UTF-8?q?fix(diarization):=20=D1=81=D0=B2=D1=8F?= =?UTF-8?q?=D0=B7=D0=B0=D0=BD=D0=B0=20=D0=BF=D0=B5=D1=80=D0=B5=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D0=B8=D0=BC=D0=B0=D1=8F=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5?= =?UTF-8?q?=D1=80=D0=BA=D0=B0=20handoff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Зачем: - капсула должна содержать каждый вход, названный в manifest - Intel handoff нужно проверять на Ryzen без сравнения чужого SKU Что: - входы manifest связаны с индексом и SHA-256 файлов капсулы - validate-only финального handoff пропускает только проверку машины - версии Python, пакетов и экспериментальной сборки по-прежнему обязательны Проверка: - uv run pytest -q tests/test_benchmark_experiment.py - uv run ruff check для изменённых файлов --- src/local_transcriber/benchmark_experiment.py | 53 +++++++++++++-- tests/test_benchmark_experiment.py | 65 ++++++++++++++++++- 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/local_transcriber/benchmark_experiment.py b/src/local_transcriber/benchmark_experiment.py index 8c8ac91..84bc07e 100644 --- a/src/local_transcriber/benchmark_experiment.py +++ b/src/local_transcriber/benchmark_experiment.py @@ -879,7 +879,10 @@ def environment_snapshot() -> dict[str, Any]: def validate_environment( - manifest: Mapping[str, Any], environment: Mapping[str, Any] + manifest: Mapping[str, Any], + environment: Mapping[str, Any], + *, + check_machine: bool = True, ) -> None: experiment = manifest["experiment"] for name, expected in experiment["dependencies"].items(): @@ -898,6 +901,8 @@ def validate_environment( } if expected_builds != {runtime_build}: raise ValueError("Хеш runtime-сборки не совпал с enabled-ячейками") + if not check_machine: + return machine = manifest["machine"] if environment.get("platform") != machine["os"]: raise ValueError("ОС не совпала с manifest") @@ -946,7 +951,9 @@ def _runtime_profile() -> str | None: except (OSError, subprocess.TimeoutExpired): return None value = completed.stdout.strip() - return value or None if completed.returncode == 0 else None + if completed.returncode != 0: + return None + return value or None def _runtime_governor() -> str | None: @@ -1640,6 +1647,32 @@ def worker_main(request_path: Path) -> None: _save_json(Path(request["response_path"]), result) +def _manifest_file_index(manifest: Mapping[str, Any]) -> dict[str, str]: + """Возвращает каждый входной файл, который manifest требует от капсулы.""" + required = {} + + def add(item: Mapping[str, Any], label: str) -> None: + path = _require_relative_path(item.get("path"), f"{label}.path") + sha256 = _require_sha256(item.get("sha256"), f"{label}.sha256") + previous = required.setdefault(path, sha256) + if previous != sha256: + raise ValueError(f"{label}: один путь ссылается на разные SHA-256") + + for artifact in manifest.get("artifacts", []): + add(artifact, f"artifact:{artifact.get('id')}") + for recording in manifest.get("recordings", []): + label = f"recording:{recording.get('id')}" + add(recording, label) + for key in ("reference", "asr_words"): + if (side := recording.get(key)) is not None: + add(side, f"{label}.{key}") + for source in manifest.get("calibration", []): + add(source, f"calibration:{source.get('id')}") + if not required: + raise ValueError("Manifest капсулы не содержит входных файлов") + return required + + def verify_capsule( archive_path: Path, expected_sha256: str, @@ -1708,6 +1741,11 @@ def verify_capsule( if declared["manifest.json"] != manifest_sha256: raise ValueError("Хеш manifest.json не совпал с индексом капсулы") archived_manifest_value = json.loads(archived_manifest) + for name, expected in _manifest_file_index(archived_manifest_value).items(): + if declared.get(name) != expected: + raise ValueError( + "Входной файл manifest не совпал с индексом файлов капсулы" + ) handoff = archived_manifest_value.get("handoff", {}) for candidate in handoff.get("candidate_recipes", []): result_file = candidate.get("result_file") @@ -1738,8 +1776,6 @@ def run_cli(args: Any, raw_manifest: Mapping[str, Any]) -> None: ) manifest = resolve_manifest_paths(raw_manifest, args.manifest.parent) warnings = validate_manifest(manifest) - environment = environment_snapshot() - validate_environment(manifest, environment) capsule_path = getattr(args, "capsule", None) capsule_sha256 = getattr(args, "capsule_sha256", None) finalized_handoff = manifest["handoff"]["outcome"] != "pending" @@ -1755,6 +1791,15 @@ def run_cli(args: Any, raw_manifest: Mapping[str, Any]) -> None: expected_report_sha256=manifest["handoff"]["public_report_sha256"], expected_capsule_id=manifest["handoff"]["capsule_id"], ) + environment = environment_snapshot() + portable_handoff_check = ( + args.validate_only and finalized_handoff and capsule_result is not None + ) + validate_environment( + manifest, + environment, + check_machine=not portable_handoff_check, + ) if args.validate_only: plan = safe_plan(manifest, warnings, environment) if capsule_result is not None: diff --git a/tests/test_benchmark_experiment.py b/tests/test_benchmark_experiment.py index fb1ed26..aa7b12f 100644 --- a/tests/test_benchmark_experiment.py +++ b/tests/test_benchmark_experiment.py @@ -294,6 +294,34 @@ def test_identity_and_runtime_validation_cover_exact_versions_and_build(tmp_path experiment.validate_environment(manifest, changed_build) +def test_portable_handoff_check_skips_only_machine_identity(tmp_path): + manifest = _manifest(tmp_path) + foreign_environment = copy.deepcopy(_environment()) + foreign_environment.update( + platform="Linux Ryzen", + logical_cores=24, + physical_cores=12, + cpu_sku="AMD Ryzen 9 5900X 12-Core Processor", + cpu_flags=["avx", "avx2", "avx512f"], + ram_bytes=64 * 1024**3, + swap_total_bytes=0, + ) + + experiment.validate_environment( + manifest, + foreign_environment, + check_machine=False, + ) + + foreign_environment["packages"]["numpy"] = "2.4.4" + with pytest.raises(ValueError, match="numpy==2.4.3"): + experiment.validate_environment( + manifest, + foreign_environment, + check_machine=False, + ) + + def test_manifest_rejects_stage_boundary_thread_budget_and_non_alternating_ab(tmp_path): manifest = _manifest(tmp_path) manifest["cells"][0]["stage"] = "ryzen" @@ -733,7 +761,19 @@ def test_capsule_verification_checks_hashes_commits_and_safe_paths(tmp_path): payload = b"private" payload_hash = hashlib.sha256(payload).hexdigest() manifest_path = tmp_path / "manifest.json" - manifest_bytes = b'{"schema_version":3}\n' + manifest_value = { + "schema_version": 3, + "artifacts": [ + { + "id": "input", + "path": "data/payload.bin", + "sha256": payload_hash, + } + ], + "recordings": [], + "calibration": [], + } + manifest_bytes = json.dumps(manifest_value).encode() manifest_path.write_bytes(manifest_bytes) capsule = { "capsule_id": "opaque-41", @@ -768,6 +808,19 @@ def test_capsule_verification_checks_hashes_commits_and_safe_paths(tmp_path): "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), } + incomplete_capsule = copy.deepcopy(capsule) + incomplete_capsule["files"].pop("data/payload.bin") + incomplete_path = tmp_path / "incomplete.zip" + with zipfile.ZipFile(incomplete_path, "w") as archive: + archive.writestr("capsule.json", json.dumps(incomplete_capsule)) + archive.writestr("manifest.json", manifest_bytes) + with pytest.raises(ValueError, match="Входной файл manifest"): + experiment.verify_capsule( + incomplete_path, + experiment.file_sha256(incomplete_path), + expected_manifest_path=manifest_path, + ) + unrelated_manifest = tmp_path / "unrelated-manifest.json" unrelated_manifest.write_bytes(b'{"schema_version":3,"other":true}\n') with pytest.raises(ValueError, match="не совпал с manifest.json капсулы"): @@ -791,9 +844,16 @@ def test_capsule_verification_checks_hashes_commits_and_safe_paths(tmp_path): def test_capsule_binds_candidate_result_evidence_to_file_index(tmp_path): + input_bytes = b"input" + input_hash = hashlib.sha256(input_bytes).hexdigest() result_bytes = b'{"aggregate":{"quality_passed":true}}' result_hash = hashlib.sha256(result_bytes).hexdigest() manifest_value = { + "artifacts": [ + {"id": "input", "path": "data/input.bin", "sha256": input_hash} + ], + "recordings": [], + "calibration": [], "handoff": { "candidate_recipes": [ { @@ -812,6 +872,7 @@ def test_capsule_binds_candidate_result_evidence_to_file_index(tmp_path): "public_report_sha256": "4" * 64, "files": { "manifest.json": hashlib.sha256(manifest_bytes).hexdigest(), + "data/input.bin": input_hash, "results/candidate.json": result_hash, }, } @@ -819,6 +880,7 @@ def test_capsule_binds_candidate_result_evidence_to_file_index(tmp_path): with zipfile.ZipFile(archive_path, "w") as archive: archive.writestr("capsule.json", json.dumps(capsule)) archive.writestr("manifest.json", manifest_bytes) + archive.writestr("data/input.bin", input_bytes) archive.writestr("results/candidate.json", result_bytes) experiment.verify_capsule( @@ -843,6 +905,7 @@ def test_capsule_binds_candidate_result_evidence_to_file_index(tmp_path): with zipfile.ZipFile(broken_path, "w") as archive: archive.writestr("capsule.json", json.dumps(capsule)) archive.writestr("manifest.json", broken_manifest_bytes) + archive.writestr("data/input.bin", input_bytes) archive.writestr("results/candidate.json", result_bytes) with pytest.raises(ValueError, match="Результат кандидата"): experiment.verify_capsule( From 42f094ccb9464742683eec9d695e5aa9a69dd4ec Mon Sep 17 00:00:00 2001 From: Dmitry Dementiev Date: Mon, 31 Aug 2026 16:49:09 +0300 Subject: [PATCH 4/4] =?UTF-8?q?docs(diarization):=20=D0=BE=D0=BF=D1=83?= =?UTF-8?q?=D0=B1=D0=BB=D0=B8=D0=BA=D0=BE=D0=B2=D0=B0=D0=BD=20=D0=BE=D1=82?= =?UTF-8?q?=D1=87=D0=B5=D1=82=20Intel-=D1=8D=D1=82=D0=B0=D0=BF=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-08-31-cpu-diarization-intel.md | 292 ++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 docs/benchmarks/2026-08-31-cpu-diarization-intel.md diff --git a/docs/benchmarks/2026-08-31-cpu-diarization-intel.md b/docs/benchmarks/2026-08-31-cpu-diarization-intel.md new file mode 100644 index 0000000..d4c9b0f --- /dev/null +++ b/docs/benchmarks/2026-08-31-cpu-diarization-intel.md @@ -0,0 +1,292 @@ +# CPU-диаризация: Intel-этап эксперимента + +**Дата:** 2026-08-31 +**Статус:** `handoff`; в Ryzen-этап передается один кандидат. Production-профиль не менялся. + +## Итог + +На Intel Core i7-6820HQ пройдены этапы 0-4 эксперимента из +[спецификации](../specs/2026-08-30-cpu-diarization-experiment.md). Для Ryzen +передается комбинация C2: + +- `window_shift_ratio=0.1`; +- Pyannote Segmentation 3.0 FP32; +- WeSpeaker ResNet34 LM FP32; +- текущий `FastClustering` с порогом `0.89`; +- два outer worker, одна общая ONNX Runtime session, общий пул из четырех + intra-op потоков, `inter_op_threads=1`. + +C2 сохранила результат baseline по всем обязательным и диагностическим +показателям: `3 / 2 / 2` кластера, та же последовательность слов, те же числа +слов без говорящего `51 / 13 / 9` и та же `mapped speaker purity`. Один прогон +трех фрагментов занял 140,632 с против 146,978 с у соседнего baseline: +ускорение 4,32%. Пиковый RSS вырос с 376,4 до 406,0 МиБ. Порог продолжения на +Intel не достигнут. Разница неотличима от наблюдаемого разброса +одиночных запусков: контрольные baseline в разных сериях лежали в диапазоне +146,978-157,892 с, то есть шире, чем разница между W0 и C2. +Спецификация при этом разрешает передать работоспособный механизм shared-session +как аппаратного кандидата для отдельного A/B на Ryzen. Результат 4,32% остается +диагностическим наблюдением; оценивать по нему ожидаемое ускорение нельзя. + +CAMPPlus + NME оказался самым быстрым вариантом: C1 заняла 98,347 с и +сохранила `3 / 2 / 2`. Но на одной записи число слов без говорящего +выросло с 51 до 53. Для такого изменения спецификация требует слуховой проверки +отмеченных мест до сборки комбинации. Материал для нее подготовлен, но в этом +окружении прослушивание недоступно. Поэтому C1 измерена и описана, но в handoff +не включена. + +## Зачем понадобился sherpa-onnx 1.13.6 + +Версия стенда поднята с `1.13.5` до `1.13.6`. В ней публичная Python-конфигурация +сегментации получила `window_shift_ratio`, необходимый для этапа W0-W4. Свежий +контрольный прогон обеих версий дал одинаковые `3 / 2 / 2` кластера, числа слов +без говорящего `51 / 13 / 9` и purity `0,76655 / 0,78742`. Каноническая сверка +интервалов и назначения говорящих словам вынесена в отдельную проверку по сырым +результатам капсулы; расхождений, требующих прослушивания, не обнаружено. Время +одного прогона составило 190,104 с для `1.13.5` и 157,892 с для `1.13.6`; это +диагностическое сравнение без повторов, а не самостоятельное доказательство +ускорения версии. + +Экспериментальные `Prepare/Finalize` и общий worker API в `1.13.6` отсутствуют. +Их нет и в проверенном актуальном upstream `master` sherpa-onnx от 2026-08-31. +Для эксперимента собран локальный патч и wheel: + +| Артефакт | SHA-256 | +|---|---| +| Патч sherpa-onnx 1.13.6 | `5f12df858cd525c8a0cfe8ff95b647c3b0e915a13d45621acc4ac42660014c49` | +| Wheel CPython 3.13, Linux x86_64 | `306499d8668ee138357f1de43a5c1c88f4dc46c845e8a92e6ad945327bcaa050` | + +Проверка патча на реальной модели подтвердила, что worker владеет подготовленным +состоянием. Копирование матрицы и одноразовая финализация работают корректно; +штатный `process()` точно совпадает с цепочкой +`Prepare -> cluster(copy) -> Finalize`. Один и два worker дали одинаковые +строки, метки и итоговые интервалы без пропусков и дублей. + +## Условия + +- Intel Core i7-6820HQ, 4 физических ядра / 8 логических процессоров; +- 32 181 526 528 байт RAM, x86_64, AVX2; VNNI и AVX-512 отсутствуют; +- Linux 7.0.0-30-generic, Python 3.13.13; +- `sherpa-onnx` 1.13.6, ONNX Runtime 1.28.0, NumPy 2.4.3; +- питание от сети, системный профиль `balanced`, governor `powersave`, turbo + включен; +- первая попытка финального W0 отбракована из-за swap-in; повторная попытка W0 + и первая попытка C2 прошли runtime guard. В принятых попытках W0 нагрел CPU + с 80 до 85 C, C2 - с 80 до 82 C; power limits, throttle count и swap I/O + не изменились; +- три контрольных фрагмента по 300 с, ожидаемые кластеры `3 / 2 / 2`; +- один запуск каждой исследовательской ячейки и одна соседняя пара W0/C2 под + runtime guard; ASR-sidecar подготовлены заранее и во время ячейки не учитывались. + +Измеряемый интервал включает полную диаризацию фрагмента. Для E3 отдельно +помечена составная оценка, потому что независимые session измерялись на заранее +подготовленных заданиях. + +## Идентификаторы прогона + +- source commit: `6345cb6449d906767b6ed33e695f2ca4eeaf3008`; +- experiment ID: `457dffbb690516a2b38ee1331a3939664ada3a0c4ade638eab6b7c5270b17f4e`; +- capsule ID: `intel-41-5de5f454`; +- SHA-256 scheduler output: + `14c5aa1d3131f9bd324c77d4c3bf315772e6374835f28863c5db45fc27ab5ce5`. + +Manifest оставляет исследовательские ячейки в матрице как уже измеренные и +выключенные, а через scheduler повторяет только финальную соседнюю пару: + +| Ячейка | Позиция | Cell ID | Статус в финальном плане | +|---|---:|---|---| +| W0 | 1 | `68360310692540e5df2640e3d653c12b91c6acaedfae56170e6ce74ecea12af2` | сырой результат сохранен | +| W1 | 2 | `e170c9a1808c8e1c6c824381dca39495740ba2526725c280e7a9aed115bc7a36` | сырой результат сохранен | +| W2 | 3 | `8e4283a6ae8bae62e7397e83870afd1d0706816e1e123c012d2b3ae795f4c7d4` | сырой результат сохранен | +| W3 | 4 | `46d81148e1c3eca15e6f8c1eacce41556db90a5e07429acf1ba4292103996845` | сырой результат сохранен | +| W4 | 5 | `eab104a305dc192847c16786de6cfa81f9a90bb9a80dc91627d439af4eb5d39d` | сырой результат сохранен | +| M0 | 10 | `206a1a7821cb29dfe86d19e45ff665ac6f143aad3a1b79419987162097973208` | сырой результат сохранен | +| M2 | 11 | `17559e0b25df2ca77a25d93b2238e9a726ac2ab123d11ec6ddcb9de51b298e9c` | сырой результат сохранен | +| M3 | 12 | `c5a5d9eb45172caeb5a4d476463f0a783d6337312aab6ef7177f2a6dfcca53d2` | сырой результат сохранен | +| Q0 | 20 | `386b2b9ba686466bf4603dda51222e7cdd6324061a68babed339db3e8e6a09dc` | сырой результат сохранен | +| Q1 | 21 | `e1cee039ce5c65ea91004df99eaefff50b02ce1c40a8f53eba34a234522a0e3f` | сырой результат сохранен | +| E1 | 30 | `4175101fcb0abbc6f4f46caa87273c431ce5fab61620493b13df4ed18f05f29d` | сырой результат сохранен | +| E0 для E1 | 31 | `9f59ab52ffd188da28f94233f13a19a297568e7c1e754c2a9e09157bca10ad85` | сырой результат сохранен | +| E0 для E2 | 32 | `c349a249896faeddc06356fd671ff2505bc2a89075ea7b2407964d6afae90232` | сырой результат сохранен | +| E2 | 33 | `1aba4e138f3cdfd675cf411b485cfd5a01c3166d4ae893a779d88c75fa1f53c2` | сырой результат сохранен | +| E3 | 34 | `6d2af928e2270eea37735ab39314b050fd520911cc07ba2303016ab657dd610a` | сырой результат сохранен | +| E0 для E3 | 35 | `c55eac6d19ea7741debb02a1b602782dc0332bafe72323ea54651845416d4b6f` | сырой результат сохранен | +| C1 | 40 | `3898cd41f6dc7d50bed245bfa6ec945c700a7fad67ac3f2e7f950d864a9ac9de` | исключена | +| W0 для C2 | 50 | `44a6ce169f9566088376b0f6139f6823783baff481edd8741e39690534c6c47d` | выполнена, baseline | +| C2 | 51 | `90ad304d332efa4742490f3a4a40cfb7e3422dfa85a22fbf254c7f4900b4a51f` | выполнена, handoff | + +TitaNet batch 1/4 остался отдельным mechanism smoke и в финальную A/B-матрицу +не входит. + +## Этап 0: механизмы + +`Prepare/Finalize`, общий worker и строгая схема manifest v3 прошли smoke-тесты. +Для CAMPPlus и TitaNet на одних и тех же подготовленных матрицах сравнивались +eigengap и NME: + +| Модель | Строки матриц | Eigengap, N | NME, N | Выбор | +|---|---|---|---|---| +| CAMPPlus | `531 / 414 / 440` | `6 / 1 / 7` | `3 / 2 / 2` | NME | +| TitaNet small | `531 / 414 / 440` | `7 / 2 / 4` | `3 / 2 / 2` | NME | + +Ни одна матрица не потеряла строки. Повторный запуск счетчика и штатная +финализация дали одинаковые параметры и метки. + +WeSpeaker S8S8 QDQ построен на 48 интервалах отдельной записи. Manifest хранит +исходный SHA-256 и точный интервал каждого элемента. Граф сохранил интерфейс, +opset и метаданные, прошел ONNX checker, динамические входные длины в ONNX +Runtime и загрузку через sherpa-onnx. SHA-256 графа: +`9a3f8ae64d1083b64a6276f1e1138b17b7511a9a8d731b632945efc84098588b`. + +## Этап 1: шаг окна + +| Ячейка | Шаг | Время, с | Кластеры | Качество | Решение | +|---|---:|---:|---|:---:|---| +| W0 | 0,10 | 148,220 | `3 / 2 / 2` | да | baseline | +| W1 | 0,15 | 100,799 | `2 / 2 / 2` | нет | исключен | +| W2 | 0,20 | 76,127 | `2 / 2 / 2` | нет | исключен | +| W3 | 0,25 | 61,369 | `2 / 2 / 2` | нет | исключен | +| W4 | 0,50 | 31,291 | `1 / 2 / 2` | нет | исключен | + +Сокращение числа окон объясняет ускорение, но все увеличенные шаги потеряли +кластер на первой записи. Рабочим остался `0.1`; C0 совпал бы с baseline и +поэтому не запускался. + +## Этапы 2-3: модели и исполнение + +| Ветвь | Время, с | Изменение к W0 | Пиковый RSS, МиБ | Кластеры | Решение | +|---|---:|---:|---:|---|---| +| M2, CAMPPlus + NME | 94,550 | -36,21% | 358,6 | `3 / 2 / 2` | самая быстрая; нужна слуховая проверка | +| M3, TitaNet + NME | 107,205 | -27,67% | 511,9 | `3 / 2 / 2` | прошел, но слабее M2 | +| QDQ WeSpeaker | 177,537 | +19,78% | 367,7 | `2 / 2 / 2` | исключен по качеству | +| E1, shared 2 x pool 4 | 139,219 | -6,07% | 398,6 | `3 / 2 / 2` | допущен как аппаратный кандидат | +| E2, shared 4 x 1 | 152,167 | +2,66% | 551,2 | `3 / 2 / 2` | исключен по времени и памяти | +| E3, 4 отдельные session | 159,038 | +7,30% | 732,3 | `3 / 2 / 2` | отрицательный контроль | + +Время E3 - составная оценка, а не прямое время штатного `process()`. RSS E3 - +максимум всего проверочного процесса: он включает штатную подготовку, +экспериментальные эмбеддинги и контрольную финализацию. Отдельный RSS только +embedding-этапа не измерялся. Даже эта завышенная оценка, включающая соседние +этапы, подтверждает ожидаемый рост памяти с числом session, поэтому E3 не стал +кандидатом. + +У E1 и E2 метки, итоговые интервалы и диагностические показатели побайтово +совпали с последовательным вариантом. E1 выбран для C2 как более быстрый и +экономный режим. + +TitaNet batch 1 воспроизвел штатные эмбеддинги с максимальным абсолютным +расхождением не больше `3,4e-6`. Batch 4 сохранил `3 / 2 / 2`, но поменял метки +строк на всех трех матрицах, дал дрейф эмбеддингов `0,277-0,317` и оказался +медленнее batch 1. Ветвь остановлена до CPU-ячейки. + +## Этап 4: комбинации + +| Комбинация | Рецепт | Время, с | Ускорение | RSS, МиБ | Диагностика | Handoff | +|---|---|---:|---:|---:|---|:---:| +| C1 | `0.1` + CAMPPlus + NME + sequential | 98,347 | 33,65% | 399,5 | purity выросла; `+2` слова без говорящего | нет | +| C2 | `0.1` + WeSpeaker FP32 + shared 2 x pool 4 | 140,632 | 4,32% | 406,0 | точное совпадение с W0 | да | + +Процент C1 посчитан относительно одиночного W0 из этапа окна, 148,220 с; это не +соседняя A/B-пара. Процент C2 посчитан относительно соседнего W0 под runtime +guard, 146,978 с. + +У C1 `mapped speaker purity` выросла с `0,76655 / 0,78742` до +`0,80137 / 0,91079`. Слова без говорящего изменились с `51 / 13 / 9` на +`53 / 13 / 9`. Без слухового решения это не считается пройденной диагностикой. + +## Закрепленные модели + +| Идентификатор | SHA-256 | +|---|---| +| Pyannote FP32 | `220ad67ca923bef2fa91f2390c786097bf305bceb5e261d4af67b38e938e1079` | +| WeSpeaker FP32 | `e9848563da86f263117134dfd7ad63c92355b37de492b55e325400c9d9c39012` | +| WeSpeaker S8S8 QDQ | `9a3f8ae64d1083b64a6276f1e1138b17b7511a9a8d731b632945efc84098588b` | +| CAMPPlus | `aa3cfc16963a10586a9393f5035d6d6b57e98d358b347f80c2a30bf4f00ceba2` | +| TitaNet small | `ad4a1802485d8b34c722d2a9d04249662f2ece5d28a7a039063ca22f515a789e` | + +## Индекс публичных доказательств + +Хеши ниже связывают обезличенные агрегаты с сырыми файлами приватной капсулы, +не публикуя пути, текст или интервалы. + +| Результат | SHA-256 | Окна по записям | Embedding-задания по записям | +|---|---|---|---| +| fresh 1.13.5 | `c53c1f970ee4db094779bc32b42f978c37c2cb74d814504ccfac87f09e802ddd` | `291 / 291 / 291` | `531 / 414 / 440` | +| fresh 1.13.6 | `eb7e3d508836e1eefd679a851271eaa6823d688fef39d2ae03978e7fa7006bba` | `291 / 291 / 291` | `531 / 414 / 440` | +| W0 | `32d513cd0e81fdca0e37484f86b5c4312d3573608493972a282302dd891cdf0e` | `291 / 291 / 291` | `531 / 414 / 440` | +| W1 | `be8935b5a81e2aadaeb38540e615c0cd3ae5ac31d6e0949856fd7d16d31a78ea` | `195 / 195 / 195` | `357 / 274 / 302` | +| W2 | `f95433d516772d40c4071da0d0745b981314ee7a7063fb4a43518e94bc17848e` | `146 / 146 / 146` | `264 / 206 / 222` | +| W3 | `14cac4f1d8cd38ed5b4a20189438d2b1e109ba267c5feaa36f0928fbec848580` | `117 / 117 / 117` | `211 / 165 / 175` | +| W4 | `6fafefce00e90590ab2d834fb560413cd7780e20cbd1f631d9954ec8b97a540e` | `59 / 59 / 59` | `105 / 82 / 89` | +| M2, исправленный интервал | `3debc8ae5cf1bd13d806dbfe8e4cfa137d6705622a721f4a42f1a22c030ec277` | `291 / 291 / 291` | `531 / 414 / 440` | +| M3, исправленный интервал | `531a842234e35e29f582ff8a51406fb9dd6211e017b3162d80cca21d4732d316` | `291 / 291 / 291` | `531 / 414 / 440` | +| E1, исправленный интервал | `f5ac41d29eeff1f7cfbb93497a0025aef40de0838e933e0f9baed30a236d6b85` | `291 / 291 / 291` | `531 / 414 / 440` | +| E2, исправленный интервал | `f2664d354ede1e4bf2c42b974513cdc845ff601c73f30ebae51456764fd8dcdc` | `291 / 291 / 291` | `531 / 414 / 440` | +| C1, исправленный интервал | `4f96e6f7363e0edb79d4034a07f4d3b6b0003a820fcf6bceff5426bb3c2caff4` | `291 / 291 / 291` | `531 / 414 / 440` | +| W0 для C2, canonical result | `e572dd057ec87514aca4617465c48ce361677ae19279efcadbbde404afc85b5d` | `291 / 291 / 291` | `531 / 414 / 440` | +| C2, canonical result | `d687074cc582e2ac379ed3375362f9480778feca02c7492a4c4be4fcc1a4f15a` | `291 / 291 / 291` | `531 / 414 / 440` | + +Для W0 под runtime guard суммарное user/system CPU time равно +`584,590 / 0,590` с, +средний RTF по трем записям - `0,16331`. Для C1 эти значения равны +`406,230 / 1,050` с и `0,10927`; для C2 под runtime guard - +`667,580 / 0,960` с и +`0,15626`. У C2 повышенное CPU time отражает конкурентную работу двух stream, +а решение принимается по wall time. + +Контрольные входы закреплены отдельно от результатов: + +| Запись | SHA-256 media | SHA-256 ASR-sidecar | SHA-256 reference | +|---|---|---|---| +| control 1 | `1057616b42e8add00e0eb975b02bdef0ec9f6cdfec6dbf488e0c60423c9b7b87` | `3e190c060b2a9b0f4145249702d66b28c26e9b8518462000d368b431e22902bb` | - | +| control 2 | `51866d247fe3eda134cdd884f707b1f1db8855b492e8bd14d2b56b62476255ed` | `f8ce6e25d0ce7999b2d3c31c7ca9fc5f721b8925334362729eed06552296bdb5` | `9098548743a86f868dfcdd906bcde34ed1ce5a935602cbe399f4bceda8685b3c` | +| control 3 | `4422f04e2771091a0648e5422d14a31dd7ca2c8eef7ed9f4a5c63f43d8ca6400` | `eca10e7b232a211ce53decfc1a449c8ce2ae482dd7bc67c7aa9a267b12a2582d` | `dc2db5925c98ea7029bf52ad9cd7cc332d0084b91118a3d58336058f619d3d9a` | + +Для CAMPPlus NME выбрал `p=111 / 54 / 104`; время счетчика составило +`3,536 / 1,488 / 1,148` с. Для TitaNet - `p=90 / 65 / 110` и +`1,084 / 0,780 / 1,112` с. Все шесть результатов дали ожидаемое N, +повторяемые метки и ноль потерянных строк. + +## Воспроизведение на Ryzen + +После проверки commit и внешнего SHA-256 приватной капсулы: + +```bash +uv sync --frozen +uv pip install --python .venv/bin/python --reinstall \ + /path/to/capsule/build/sherpa_onnx-1.13.6-cp313-cp313-linux_x86_64.whl +export LOCAL_TRANSCRIBER_EXPERIMENT_BUILD_SHA256=\ +306499d8668ee138357f1de43a5c1c88f4dc46c845e8a92e6ad945327bcaa050 +uv run python scripts/benchmarks/diarization_calibration.py \ + --capsule /path/to/original-capsule.zip \ + --capsule-sha256 CAPSULE_SHA256_FROM_HANDOFF \ + --manifest /path/to/capsule/manifest.json \ + --validate-only +``` + +`--validate-only` должен вернуть schema v3, этап `intel`, пустой список +предупреждений и `qdq_available=true`. Затем Ryzen-агент создает свой manifest +этапа `ryzen`, указывает Intel commit как `handoff_commit` и выполняет +изолированный E1 A/B перед первичным A/B C2. Экспериментальный wheel совместим +только с CPython 3.13 на Linux x86_64; на другой платформе его нужно пересобрать +из патча с тем же SHA-256. + +Приватная капсула содержит manifest, исходные записи, ASR-sidecar, опорную +разметку, калибровочные данные, созданные модельные артефакты, сырые результаты, +worker requests и журналы. Эти файлы не входят в Git и не передаются внешним +ревьюерам. + +## Ограничения + +- У каждой ячейки один прогон. Проценты не оценивают дисперсию, прогрев и + температурный дрейф. +- Ранние ячейки запускались не чередующимися парами, поэтому полного A/B не + образуют. Финальные механизмы на Ryzen следует проверять только чередующимися + парами из спецификации. +- Сравнение sherpa-onnx 1.13.5 и 1.13.6 показывает равенство качества, но разницу + времени нельзя приписывать одной версии без повторов. +- C1 нельзя возвращать в матрицу без слуховой проверки подготовленных мест. +- Полные записи и приемка пяти слуховых категорий относятся к Ryzen-этапу после + выбора финалиста. +- Вывод относится к одному Intel Core i7-6820HQ без VNNI/AVX-512. Результаты + QDQ и shared-session нельзя переносить на Ryzen без собственного A/B.