Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat(benchmark): report first nonempty transcript latency
LauraGPT99ef3b727b57
diff --git a/.github/workflows/test-realtime-benchmark.yml b/.github/workflows/test-realtime-benchmark.ymlnew file mode 100644index 000000000..9ef63ed79--- /dev/null+++ b/.github/workflows/test-realtime-benchmark.yml@@ -0,0 +1,35 @@+name: Validate realtime benchmark++on:+ pull_request:+ paths:+ - "examples/industrial_data_pretraining/fun_asr_nano/realtime_ws_benchmark.py"+ - "tests/test_realtime_ws_benchmark.py"+ - "docs/benchmark/realtime_ws_benchmark.md"+ - ".github/workflows/test-realtime-benchmark.yml"+ push:+ branches:+ - main+ paths:+ - "examples/industrial_data_pretraining/fun_asr_nano/realtime_ws_benchmark.py"+ - "tests/test_realtime_ws_benchmark.py"+ - "docs/benchmark/realtime_ws_benchmark.md"+ - ".github/workflows/test-realtime-benchmark.yml"++permissions:+ contents: read++jobs:+ benchmark:+ runs-on: ubuntu-latest+ timeout-minutes: 5+ steps:+ - uses: actions/checkout@v4+ - uses: actions/setup-python@v5+ with:+ python-version: "3.11"+ cache: pip+ - name: Install benchmark test dependencies+ run: python -m pip install pytest websockets+ - name: Run CPU-only benchmark tests+ run: python -m pytest -q tests/test_realtime_ws_benchmark.pydiff --git a/docs/benchmark/realtime_ws_benchmark.md b/docs/benchmark/realtime_ws_benchmark.mdindex 6cf8f57f7..71eaf77bc 100644--- a/docs/benchmark/realtime_ws_benchmark.md+++ b/docs/benchmark/realtime_ws_benchmark.md@@ -75,12 +75,23 @@ stress signal, not as user-facing realtime latency.|--------|---------|| `aggregate_audio_per_wall` | Total input audio seconds across all clients divided by benchmark wall time || `first_update_ms_p50/p95` | Time from first audio frame to first result message with `sentences`, `partial`, or `is_final` |+| `first_text_ms_p50/p95` | Time from first audio frame to first result containing non-blank `partial` or sentence `text`, including final-only text |+| `clients_with_text` | Number of clients that received non-blank text; compare with `clients` when interpreting first-text percentiles || `final_after_stop_ms_p50/p95` | Time from sending `STOP` to receiving the final result || `client_response_lag_ms_p95_max` | Largest per-client p95 of non-final `(client receive time - audio start) - server duration_ms`; useful mainly in paced mode for preview/partial lag || `partial_messages` | Count of non-final result messages with a non-empty `partial` || `final_messages` | Count of final result messages || `errors` | Connection, timeout, protocol, or client-side validation errors |+`first_update_ms` is unchanged: an empty result message can set it before any+transcript arrives. Use `first_text_ms` for the first client-visible transcript,+not model token-generation latency or a guarantee that the text is correct or+stable. Each client JSONL row includes both values. A client with no non-blank+text has `first_text_ms: null` and is excluded from the first-text percentiles;+if no client receives text, both percentiles are `null`. Always report+`clients_with_text`, total `clients`, and `errors` together so missing transcripts+cannot look like a latency improvement.+The script can observe only client-side timing and fields returned by theserver. For a performance investigation, add `--log-decode-profile` to recordone structured line per engine call with the request and sample counts, audiodiff --git a/examples/industrial_data_pretraining/fun_asr_nano/realtime_ws_benchmark.py b/examples/industrial_data_pretraining/fun_asr_nano/realtime_ws_benchmark.pyindex 0bd2c37e1..1068d9c72 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/realtime_ws_benchmark.py+++ b/examples/industrial_data_pretraining/fun_asr_nano/realtime_ws_benchmark.py@@ -103,6 +103,13 @@ async def recv_results(ws, metrics, audio_started_at, stop_sent_at_ref, timeout)metrics["result_messages"] += 1if metrics["first_update_ms"] is None:metrics["first_update_ms"] = (now - audio_started_at) * 1000.0+ texts = [data.get("partial")] + [+ sentence.get("text") for sentence in data.get("sentences", [])+ ]+ if metrics["first_text_ms"] is None and any(+ isinstance(text, str) and text.strip() for text in texts+ ):+ metrics["first_text_ms"] = (now - audio_started_at) * 1000.0if data.get("partial"):metrics["partial_messages"] += 1duration_ms = data.get("duration_ms")@@ -135,6 +142,7 @@ async def run_client(client_id, args, audio_bytes, audio_seconds):"final_messages": 0,"events": {},"first_update_ms": None,+ "first_text_ms": None,"final_update_ms": None,"final_after_stop_ms": None,"response_lag_ms": [],@@ -194,6 +202,7 @@ async def run_client(client_id, args, audio_bytes, audio_seconds):metrics.update({"first_update_ms": round_or_none(metrics["first_update_ms"], 1),+ "first_text_ms": round_or_none(metrics["first_text_ms"], 1),"final_update_ms": round_or_none(metrics["final_update_ms"], 1),"final_after_stop_ms": round_or_none(metrics["final_after_stop_ms"], 1),"wall_seconds": round(wall_seconds, 3),@@ -210,6 +219,7 @@ async def run_client(client_id, args, audio_bytes, audio_seconds):def summarize(results, elapsed_seconds):total_audio = sum(item["audio_seconds"] for item in results)first_updates = [item["first_update_ms"] for item in results if item["first_update_ms"] is not None]+ first_texts = [item["first_text_ms"] for item in results if item["first_text_ms"] is not None]final_after_stop = [item["final_after_stop_ms"] for item in results if item["final_after_stop_ms"] is not None]@@ -224,6 +234,11 @@ def summarize(results, elapsed_seconds):if first_updateselse None,"first_update_ms_p95": round_or_none(percentile(first_updates, 95), 1),+ "first_text_ms_p50": round_or_none(statistics.median(first_texts), 1)+ if first_texts+ else None,+ "first_text_ms_p95": round_or_none(percentile(first_texts, 95), 1),+ "clients_with_text": len(first_texts),"final_after_stop_ms_p50": round_or_none(statistics.median(final_after_stop), 1)if final_after_stopelse None,@@ -242,6 +257,8 @@ def print_summary(summary, results):print(f"elapsed seconds: {summary['elapsed_seconds']}")print(f"aggregate audio/wall: {summary['aggregate_audio_per_wall']}x")print(f"first update p50/p95 ms: {summary['first_update_ms_p50']} / {summary['first_update_ms_p95']}")+ print(f"first text p50/p95 ms: {summary['first_text_ms_p50']} / {summary['first_text_ms_p95']}")+ print(f"clients with text: {summary['clients_with_text']}/{summary['clients']}")print("final after STOP p50/p95 ms: "f"{summary['final_after_stop_ms_p50']} / {summary['final_after_stop_ms_p95']}"diff --git a/tests/test_realtime_ws_benchmark.py b/tests/test_realtime_ws_benchmark.pyindex f1224ea42..cae0ee30d 100644--- a/tests/test_realtime_ws_benchmark.py+++ b/tests/test_realtime_ws_benchmark.py@@ -3,6 +3,10 @@ import importlib.utilimport jsonimport sysfrom pathlib import Path+from types import SimpleNamespace+import wave++import pytestREPO_ROOT = Path(__file__).resolve().parents[1]@@ -48,6 +52,7 @@ def test_final_message_does_not_contribute_to_response_lag(monkeypatch):"final_messages": 0,"events": {},"first_update_ms": None,+ "first_text_ms": None,"final_update_ms": None,"final_after_stop_ms": None,"response_lag_ms": [],@@ -123,3 +128,145 @@ def test_client_ping_timeout_zero_disables_timeout():args = module.parse_args(["audio.wav", "--client-ping-timeout", "0"])assert args.client_ping_timeout is None+++ "messages,expected",+ [+ ([{"partial": ""}, {"partial": "hello"}, {"partial": "later"}], 1000.0),+ ([{"sentences": []}, {"sentences": [{"text": "hello"}]}], 1000.0),+ (+ [{"partial": ""}, {"is_final": True, "sentences": [{"text": "hello"}]}],+ 1000.0,+ ),+ (+ [{"partial": " \t", "sentences": [{"text": "\n"}]}, {"partial": "hello"}],+ 1000.0,+ ),+ ([{"partial": "hello"}, {"partial": "later"}], 0.0),+ ([{"sentences": []}, {"is_final": True, "sentences": []}], None),+ ([{"partial": None, "sentences": [{"text": None}]}], None),+ ],+)+def test_first_text_waits_for_nonblank_transcript(monkeypatch, messages, expected):+ module = load_benchmark_module()+ responses = iter([*messages, {"event": "stopped"}])+ timestamps = iter(float(i) for i in range(len(messages) + 1))++ async def receive_message(_ws, _timeout):+ return next(responses)++ monkeypatch.setattr(module, "receive_message", receive_message)+ monkeypatch.setattr(+ module, "time", SimpleNamespace(perf_counter=lambda: next(timestamps))+ )+ metrics = {+ "messages": 0,+ "result_messages": 0,+ "partial_messages": 0,+ "final_messages": 0,+ "events": {},+ "first_update_ms": None,+ "first_text_ms": None,+ "final_update_ms": None,+ "final_after_stop_ms": None,+ "response_lag_ms": [],+ "stopped": False,+ "errors": [],+ }+ asyncio.run(module.recv_results(object(), metrics, 0.0, {"value": None}, 1.0))++ assert metrics["first_update_ms"] == 0.0+ assert metrics["first_text_ms"] == expected+ assert metrics["stopped"] is True+ assert metrics["errors"] == []+++ "latencies,expected_p50,expected_p95,count",+ [+ ([None, 0.0, 1000.0, 2000.0], 1000.0, 1900.0, 3),+ ([None, None], None, None, 0),+ ],+)+def test_first_text_summary_excludes_missing_text_but_counts_coverage(+ latencies, expected_p50, expected_p95, count+):+ module = load_benchmark_module()+ results = [+ {+ "audio_seconds": 1.0,+ "first_update_ms": 0.0,+ "first_text_ms": latency,+ "final_after_stop_ms": 0.0,+ "response_lag_ms_p95": 0.0,+ "partial_messages": 0,+ "final_messages": 1,+ "errors": [],+ }+ for latency in latencies+ ]+ summary = module.summarize(results, 3.0)++ assert summary["first_text_ms_p50"] == expected_p50+ assert summary["first_text_ms_p95"] == expected_p95+ assert summary["clients_with_text"] == count+ assert summary["clients"] == len(latencies)+ assert summary["first_update_ms_p50"] == 0.0+++def test_first_text_round_trip_exports_client_and_summary_jsonl(tmp_path, capsys):+ module = load_benchmark_module()+ wav_path = tmp_path / "silence.wav"+ output_path = tmp_path / "metrics.jsonl"+ with wave.open(str(wav_path), "wb") as audio:+ audio.setnchannels(1)+ audio.setsampwidth(2)+ audio.setframerate(16000)+ audio.writeframes(b"\0\0" * 3200)++ async def replay_server(ws, *_args):+ assert await ws.recv() == "START"+ await ws.send(json.dumps({"event": "started"}))+ assert isinstance(await ws.recv(), bytes)+ await ws.send(json.dumps({"sentences": [], "partial": ""}))+ assert isinstance(await ws.recv(), bytes)+ await ws.send(json.dumps({"partial": "hello"}))+ assert await ws.recv() == "STOP"+ await ws.send(json.dumps({"is_final": True, "sentences": [{"text": "hello"}]}))+ await ws.send(json.dumps({"event": "stopped"}))++ async def run():+ async with module.websockets.serve(replay_server, "127.0.0.1", 0) as server:+ port = server.sockets[0].getsockname()[1]+ args = module.parse_args(+ [+ str(wav_path),+ "--server",+ f"ws://127.0.0.1:{port}",+ "--no-pace",+ "--clients",+ "2",+ "--output-jsonl",+ str(output_path),+ ]+ )+ return await asyncio.wait_for(module.async_main(args), timeout=5)++ assert asyncio.run(run()) == 0+ records = [json.loads(line) for line in output_path.read_text().splitlines()]+ clients = [row for row in records if row["type"] == "client"]+ summary = records[-1]+ assert len(clients) == 2+ for client in clients:+ assert client["errors"] == []+ assert 0 <= client["first_update_ms"] <= client["first_text_ms"]+ assert client["first_text_ms"] == round(client["first_text_ms"], 1)+ assert client["stopped"] is True+ assert summary["type"] == "summary"+ assert summary["clients_with_text"] == summary["clients"] == 2+ assert summary["first_text_ms_p50"] is not None+ assert summary["first_text_ms_p95"] is not None+ printed = capsys.readouterr().out+ assert "first text p50/p95 ms:" in printed+ assert "clients with text: 2/2" in printed