Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

9.3 KB
Raw Blame
"""Check documented contracts against source without importing model runtimes."""

import ast
import hashlib
import inspect
import json
import re
from pathlib import Path

import pytest


ROOT = Path(__file__).resolve().parents[1]
GUIDES = ("vllm_guide.md", "vllm_guide_zh.md", "vllm_guide_zh_v2.md")
NANO = "funasr/models/fun_asr_nano/inference_vllm.py"
AUTO = "funasr/auto/auto_model_vllm.py"
VAD = "funasr/models/fsmn_vad_streaming/model.py"
SERVER = "examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py"


def blocks(text, language):
    return re.findall(rf"^```{language}\n(.*?)^```", text, re.M | re.S)


@pytest.fixture(params=GUIDES)
def guide(request):
    return request.param, (ROOT / "docs" / request.param).read_text()


def source_function(path, name, class_name=None):
    tree = ast.parse((ROOT / path).read_text())
    if class_name:
        tree = next(
            node
            for node in tree.body
            if isinstance(node, ast.ClassDef) and node.name == class_name
        )
    return next(
        node
        for node in tree.body
        if isinstance(node, ast.FunctionDef) and node.name == name
    )


def signature(path, class_name):
    function = source_function(path, "generate", class_name)
    function.body = [ast.Pass()]
    function.decorator_list = []
    function.returns = None
    for argument in ast.walk(function.args):
        if isinstance(argument, ast.arg):
            argument.annotation = None
    namespace = {}
    exec(
        compile(
            ast.fix_missing_locations(ast.Module(body=[function], type_ignores=[])),
            path,
            "exec",
        ),
        namespace,
    )
    return inspect.signature(namespace["generate"])


def generate_calls(text):
    kinds = {}
    for block in blocks(text, "python"):
        tree = ast.parse(block)
        for node in ast.walk(tree):
            if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call):
                continue
            constructor = node.value.func
            if isinstance(constructor, ast.Attribute):
                constructor = constructor.value
            if not isinstance(constructor, ast.Name):
                continue
            kind = constructor.id
            if kind == "AutoModel":
                model = next(
                    (kw.value for kw in node.value.keywords if kw.arg == "model"), None
                )
                kind = (
                    "vad"
                    if isinstance(model, ast.Constant) and model.value == "fsmn-vad"
                    else kind
                )
            for target in node.targets:
                if isinstance(target, ast.Name):
                    kinds[target.id] = kind
        for node in ast.walk(tree):
            if (
                isinstance(node, ast.Call)
                and isinstance(node.func, ast.Attribute)
                and node.func.attr == "generate"
                and isinstance(node.func.value, ast.Name)
            ):
                yield kinds.get(node.func.value.id), node


def test_generate_examples_bind_to_actual_signatures(guide):
    _, text = guide
    signatures = {
        "AutoModelVLLM": signature(AUTO, "AutoModelVLLM"),
        "FunASRNanoVLLM": signature(NANO, "FunASRNanoVLLM"),
        "vad": signature("funasr/auto/auto_model.py", "AutoModel"),
    }
    checked = 0
    for kind, call in generate_calls(text):
        if kind in signatures:
            signatures[kind].bind(
                object(),
                *[object() for _ in call.args],
                **{kw.arg: object() for kw in call.keywords if kw.arg},
            )
            checked += 1
    assert checked >= 3


def test_manifest_is_a_cli_contract_not_a_direct_engine_input(guide):
    _, text = guide
    for kind, call in generate_calls(text):
        if kind not in {"AutoModelVLLM", "FunASRNanoVLLM"}:
            continue
        inputs = list(call.args) + [
            kw.value for kw in call.keywords if kw.arg == "inputs"
        ]
        for value in inputs:
            for node in ast.walk(value):
                if isinstance(node, ast.Constant) and isinstance(node.value, str):
                    assert not node.value.endswith((".scp", ".jsonl")), ast.unparse(
                        call
                    )
    assert "demo_vllm.py --input wav.scp" in text
    demo = (
        ROOT / "examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py"
    ).read_text()
    assert 'args.input.endswith(".scp")' in demo
    assert 'args.input.endswith(".jsonl")' in demo
    assert 'audio_files.append(item["source"])' in demo
    assert "load_audio_text_image_video(audio_input" in (ROOT / NANO).read_text()


def test_dynamic_options_are_sent_to_the_actual_vad_consumer(guide):
    _, text = guide
    options = {"silence_schedule", "dynamic_silence"}
    seen = set()
    for kind, call in generate_calls(text):
        specified = options.intersection(kw.arg for kw in call.keywords)
        if specified:
            assert kind == "vad", "VAD options do not configure the ASR engine"
            seen.update(specified)
    assert seen == options
    inference = source_function(VAD, "inference", "FsmnVADStreaming")
    consumed = {
        node.args[0].value
        for node in ast.walk(inference)
        if isinstance(node, ast.Call)
        and isinstance(node.func, ast.Attribute)
        and node.func.attr == "get"
        and node.args
        and isinstance(node.args[0], ast.Constant)
    }
    assert options <= consumed


def test_dynamic_schedule_table_matches_sdk_constants(guide):
    _, text = guide
    tree = ast.parse((ROOT / VAD).read_text())
    schedules = {}
    for node in tree.body:
        if (
            isinstance(node, ast.Assign)
            and isinstance(node.targets[0], ast.Name)
            and node.targets[0].id
            in {"DEFAULT_SILENCE_SCHEDULE", "STREAMING_SILENCE_SCHEDULE"}
        ):
            schedules[node.targets[0].id] = eval(
                compile(ast.Expression(node.value), VAD, "eval"),
                {"__builtins__": {}, "float": float},
            )
    rows = re.findall(r"^\| (\d+) ms \| (\d+) ms \| (\d+) ms \|$", text, re.M)
    assert len(rows) >= 9
    for duration, offline, streaming in rows:
        for column, name in [
            (offline, "DEFAULT_SILENCE_SCHEDULE"),
            (streaming, "STREAMING_SILENCE_SCHEDULE"),
        ]:
            expected = next(
                threshold
                for limit, threshold in schedules[name]
                if int(duration) <= limit
            )
            assert int(column) == expected


def test_json_examples_are_valid_and_verbose_output_matches_serializer(guide):
    _, text = guide
    examples = [json.loads(block) for block in blocks(text, "json")]
    assert len(examples) >= 2
    function = source_function(SERVER, "build_openai_verbose_json")
    namespace = {}
    exec(
        compile(ast.Module(body=[function], type_ignores=[]), SERVER, "exec"), namespace
    )
    verbose = next(
        example for example in examples if example.get("task") == "transcribe"
    )
    result = {key: verbose[key] for key in ("text", "duration", "segments")}
    assert (
        namespace["build_openai_verbose_json"](result, language=verbose["language"])
        == verbose
    )
    for example in examples:
        for segment in example.get("segments", []):
            assert 0 <= segment["start"] <= segment["end"] <= example["duration"]
            for word in segment.get("words", []):
                assert (
                    segment["start"] <= word["start"] <= word["end"] <= segment["end"]
                )


def test_websocket_examples_are_individual_messages(guide):
    _, text = guide
    streams = [
        block for block in blocks(text, "text") if '{"event": "started"}' in block
    ]
    assert len(streams) == 2
    for stream in streams:
        events = [json.loads(line) for line in stream.splitlines() if line.strip()]
        assert events[0] == {"event": "started"}
        assert events[-1] == {"event": "stopped"}
        final = next(event for event in events if event.get("is_final"))
        assert final["sentences"]
        for sentence in final["sentences"]:
            assert isinstance(sentence["text"], str) and sentence["text"] != "..."
            assert 0 <= sentence["start"] <= sentence["end"]


def test_performance_claims_distinguish_throughput_and_measured_accuracy(guide):
    _, text = guide
    assert "340 / 21" in text and "16.2" in text
    assert "8.20%" in text and "8.06%" in text and "0.14" in text
    assert "8.14%" in text and "8.19%" in text
    assert "benchmark/rtf_reproducibility.md" in text
    for stale in (
        "16-340x",
        "16–340x",
        "21.7x",
        "7.6x",
        "RTFx 340+",
        "CER 完全一致",
        "CER exactly",
        "CER 不变",
        "CER is unchanged",
        "Subsequent inferences are instant",
        "之后即时",
    ):
        assert stale not in text


def test_historical_heading_anchors_are_preserved(guide):
    name, text = guide
    prose = re.sub(r"^```[^\n]*\n.*?^```\s*$", "", text, flags=re.M | re.S)
    headings = re.findall(r"^#{1,6} .+$", prose, re.M)
    expected = (
        "678bb9b4403454e8773e1470e7e90e9ed2cf2500266eaa06c58b6dda20b48134"
        if name == "vllm_guide.md"
        else "8b56033dfcfc2bf80e12cc8fa272a313311a1a0730f8f5c0c73fd95e8c56018f"
    )
    assert hashlib.sha256("\n".join(headings).encode()).hexdigest() == expected