Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat(runtime): add GGUF HTTP transcription wrapper
zhifu gaob798e2679f9b
diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.mdindex f38460507..6f7cd62e2 100644--- a/runtime/llama.cpp/README.md+++ b/runtime/llama.cpp/README.md@@ -107,6 +107,43 @@ The shared **FSMN-VAD** front end builds the same way (`funasr-vad/` + `funasr-ctarget `llama-funasr-vad`); export weights with `export_vad_gguf.py`. Pass`--vad fsmn-vad.gguf` to any of the three tools for built-in long-audio segmentation.+## Lightweight HTTP server++The GGUF binaries are command-line tools first. For local apps that expect an+HTTP transcription endpoint, `server/funasr_gguf_server.py` wraps an existing+binary and exposes an OpenAI-compatible `POST /v1/audio/transcriptions` route.+It uses only the Python standard library and still runs inference in the C+++binary:++```bash+python server/funasr_gguf_server.py \+ --host 127.0.0.1 --port 8000 \+ --binary ./build/bin/llama-funasr-sensevoice \+ --model ./gguf/sensevoice-small-q8.gguf \+ --vad ./gguf/fsmn-vad.gguf+```++Then send audio with the same shape used by OpenAI-compatible clients:++```bash+curl http://127.0.0.1:8000/v1/audio/transcriptions \+ -F [email protected] \+ -F model=funasr-gguf+```++Response:++```json+{"text": "transcribed text"}+```++CUDA-enabled SenseVoice builds can be selected with `--backend cuda`. Extra+binary flags can be forwarded with repeated `--extra-arg`, for example+`--extra-arg --keep-tags`. This wrapper starts one subprocess per request, so it+is best for local tools, demos, and integration tests. For sustained production+traffic, use the Python `funasr-server` OpenAI-compatible service or build a+dedicated native server around the C++ runtime.+## ValidationEach model was validated against the FunASR PyTorch reference (encoder cosine ≈ 1.0;diff --git a/runtime/llama.cpp/server/funasr_gguf_server.py b/runtime/llama.cpp/server/funasr_gguf_server.pynew file mode 100644index 000000000..6c6463cf1--- /dev/null+++ b/runtime/llama.cpp/server/funasr_gguf_server.py@@ -0,0 +1,230 @@+#!/usr/bin/env python3+"""OpenAI-compatible HTTP wrapper for FunASR llama.cpp / GGUF binaries.++This server intentionally keeps inference in the existing C++ command-line+tools. It accepts a multipart audio upload, runs the configured GGUF binary, and+returns the transcript as a small OpenAI-compatible JSON response.+"""++from __future__ import annotations++import argparse+from dataclasses import dataclass, field+from email import policy+from email.parser import BytesParser+import json+from http import HTTPStatus+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+import os+from pathlib import Path+import subprocess+import tempfile+from typing import Iterable, Optional+from urllib.parse import urlparse+++@dataclass(frozen=True)+class ServerConfig:+ binary: str+ model: str+ vad: Optional[str] = None+ backend: Optional[str] = None+ extra_args: list[str] = field(default_factory=list)+ work_dir: str = field(default_factory=tempfile.gettempdir)+ timeout: float = 600.0+++def build_command(config: ServerConfig, audio_path: str) -> list[str]:+ command = [config.binary, "-m", config.model, "-a", audio_path]+ if config.vad:+ command.extend(["--vad", config.vad])+ if config.backend:+ command.extend(["--backend", config.backend])+ command.extend(config.extra_args)+ return command+++def extract_transcript(stdout: str) -> str:+ return stdout.strip()+++def transcribe_file(config: ServerConfig, audio_path: str) -> str:+ completed = subprocess.run(+ build_command(config, audio_path),+ check=True,+ capture_output=True,+ text=True,+ timeout=config.timeout,+ )+ return extract_transcript(completed.stdout)+++def _json_bytes(payload: dict) -> bytes:+ return json.dumps(payload, ensure_ascii=False).encode("utf-8")+++def _error_payload(message: str) -> bytes:+ return _json_bytes({"error": {"message": message}})+++def _field_content_disposition(part) -> str:+ return part.get("Content-Disposition", "")+++def _is_file_field(part) -> bool:+ disposition = _field_content_disposition(part)+ return "form-data" in disposition and 'name="file"' in disposition+++def parse_multipart_file(content_type: str, body: bytes) -> tuple[bytes, str]:+ message = BytesParser(policy=policy.default).parsebytes(+ (+ f"Content-Type: {content_type}\r\n"+ "MIME-Version: 1.0\r\n"+ "\r\n"+ ).encode("utf-8")+ + body+ )+ if not message.is_multipart():+ raise ValueError("expected multipart/form-data")++ for part in message.iter_parts():+ if not _is_file_field(part):+ continue+ filename = part.get_filename() or "audio.wav"+ payload = part.get_payload(decode=True) or b""+ if not payload:+ raise ValueError("uploaded file is empty")+ return payload, filename+ raise ValueError("missing multipart field: file")+++def _suffix_from_filename(filename: str) -> str:+ suffix = Path(filename).suffix+ if not suffix or len(suffix) > 16:+ return ".wav"+ return suffix+++class FunASRGGUFHandler(BaseHTTPRequestHandler):+ server_version = "FunASRGGUFServer/0.1"++ def log_message(self, fmt: str, *args) -> None:+ print("%s - - [%s] %s" % (self.address_string(), self.log_date_time_string(), fmt % args))++ @property+ def config(self) -> ServerConfig:+ return self.server.config # type: ignore[attr-defined]++ def _send_json(self, status: HTTPStatus, payload: bytes) -> None:+ self.send_response(status)+ self.send_header("Content-Type", "application/json; charset=utf-8")+ self.send_header("Content-Length", str(len(payload)))+ self.end_headers()+ self.wfile.write(payload)++ def do_GET(self) -> None:+ if urlparse(self.path).path == "/health":+ self._send_json(HTTPStatus.OK, _json_bytes({"status": "ok"}))+ return+ self._send_json(HTTPStatus.NOT_FOUND, _error_payload("not found"))++ def do_POST(self) -> None:+ if urlparse(self.path).path != "/v1/audio/transcriptions":+ self._send_json(HTTPStatus.NOT_FOUND, _error_payload("not found"))+ return++ try:+ length = int(self.headers.get("Content-Length", "0"))+ content_type = self.headers.get("Content-Type", "")+ audio_bytes, filename = parse_multipart_file(content_type, self.rfile.read(length))+ except Exception as exc:+ self._send_json(HTTPStatus.BAD_REQUEST, _error_payload(str(exc)))+ return++ tmp_path = None+ try:+ os.makedirs(self.config.work_dir, exist_ok=True)+ with tempfile.NamedTemporaryFile(+ "wb",+ suffix=_suffix_from_filename(filename),+ dir=self.config.work_dir,+ delete=False,+ ) as tmp:+ tmp.write(audio_bytes)+ tmp_path = tmp.name+ transcript = transcribe_file(self.config, tmp_path)+ except subprocess.CalledProcessError as exc:+ message = exc.stderr.strip() or exc.stdout.strip() or str(exc)+ self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, _error_payload(message))+ return+ except subprocess.TimeoutExpired:+ self._send_json(HTTPStatus.GATEWAY_TIMEOUT, _error_payload("transcription timed out"))+ return+ except Exception as exc:+ self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, _error_payload(str(exc)))+ return+ finally:+ if tmp_path:+ try:+ os.unlink(tmp_path)+ except FileNotFoundError:+ pass++ self._send_json(HTTPStatus.OK, _json_bytes({"text": transcript}))+++class FunASRHTTPServer(ThreadingHTTPServer):+ def __init__(self, server_address, handler_class, config: ServerConfig):+ super().__init__(server_address, handler_class)+ self.config = config+++def create_server(host: str, port: int, config: ServerConfig) -> FunASRHTTPServer:+ return FunASRHTTPServer((host, port), FunASRGGUFHandler, config)+++def _parse_extra_args(values: Optional[Iterable[str]]) -> list[str]:+ args: list[str] = []+ for value in values or []:+ args.extend(value.split())+ return args+++def build_arg_parser() -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(description="Serve a FunASR llama.cpp / GGUF binary over HTTP.")+ parser.add_argument("--host", default="127.0.0.1")+ parser.add_argument("--port", type=int, default=8000)+ parser.add_argument("--binary", required=True, help="Path to llama-funasr-sensevoice or llama-funasr-paraformer.")+ parser.add_argument("--model", required=True, help="Path to the model GGUF passed as -m.")+ parser.add_argument("--vad", help="Optional FSMN-VAD GGUF passed as --vad.")+ parser.add_argument("--backend", choices=["cpu", "cuda"], help="Optional backend passed as --backend.")+ parser.add_argument("--work-dir", default=tempfile.gettempdir(), help="Directory for temporary uploaded audio files.")+ parser.add_argument("--timeout", type=float, default=600.0, help="Per-request subprocess timeout in seconds.")+ parser.add_argument(+ "--extra-arg",+ action="append",+ default=[],+ help="Additional argument(s) forwarded to the GGUF binary. May be repeated.",+ )+ return parser+++def main() -> None:+ args = build_arg_parser().parse_args()+ config = ServerConfig(+ binary=args.binary,+ model=args.model,+ vad=args.vad,+ backend=args.backend,+ extra_args=_parse_extra_args(args.extra_arg),+ work_dir=args.work_dir,+ timeout=args.timeout,+ )+ httpd = create_server(args.host, args.port, config)+ print(f"Serving FunASR GGUF transcription on http://{args.host}:{httpd.server_port}")+ httpd.serve_forever()+++if __name__ == "__main__":+ main()diff --git a/runtime/llama.cpp/tests/test_gguf_server.py b/runtime/llama.cpp/tests/test_gguf_server.pynew file mode 100644index 000000000..f24a5cf12--- /dev/null+++ b/runtime/llama.cpp/tests/test_gguf_server.py@@ -0,0 +1,100 @@+import importlib.util+import json+import os+import sys+import threading+import urllib.request+from pathlib import Path+++ROOT = Path(__file__).resolve().parents[1]+SERVER = ROOT / "server" / "funasr_gguf_server.py"+++def load_server_module():+ spec = importlib.util.spec_from_file_location("funasr_gguf_server", SERVER)+ module = importlib.util.module_from_spec(spec)+ sys.modules["funasr_gguf_server"] = module+ spec.loader.exec_module(module)+ return module+++def test_build_command_adds_model_audio_vad_backend_and_extra_args(tmp_path):+ server = load_server_module()+ cfg = server.ServerConfig(+ binary="/opt/funasr/llama-funasr-sensevoice",+ model="/models/sensevoice.gguf",+ vad="/models/fsmn-vad.gguf",+ backend="cuda",+ extra_args=["--keep-tags"],+ work_dir=str(tmp_path),+ )++ command = server.build_command(cfg, "/tmp/request.wav")++ assert command == [+ "/opt/funasr/llama-funasr-sensevoice",+ "-m",+ "/models/sensevoice.gguf",+ "-a",+ "/tmp/request.wav",+ "--vad",+ "/models/fsmn-vad.gguf",+ "--backend",+ "cuda",+ "--keep-tags",+ ]+++def test_transcription_endpoint_runs_binary_and_returns_openai_json(tmp_path):+ server = load_server_module()+ fake_binary = tmp_path / "fake_funasr.py"+ captured_args = tmp_path / "args.json"+ fake_binary.write_text(+ "#!/usr/bin/env python3\n"+ "import json, pathlib, sys\n"+ f"pathlib.Path({str(captured_args)!r}).write_text(json.dumps(sys.argv[1:]))\n"+ "print('hello from gguf')\n",+ encoding="utf-8",+ )+ fake_binary.chmod(0o755)++ cfg = server.ServerConfig(+ binary=str(fake_binary),+ model=str(tmp_path / "sensevoice.gguf"),+ vad=str(tmp_path / "fsmn-vad.gguf"),+ backend=None,+ extra_args=["--ids"],+ work_dir=str(tmp_path),+ )+ httpd = server.create_server("127.0.0.1", 0, cfg)+ thread = threading.Thread(target=httpd.serve_forever, daemon=True)+ thread.start()+ try:+ boundary = "----funasr-test-boundary"+ body = (+ f"--{boundary}\r\n"+ 'Content-Disposition: form-data; name="file"; filename="sample.wav"\r\n'+ "Content-Type: audio/wav\r\n\r\n"+ "RIFFfake-audio\r\n"+ f"--{boundary}--\r\n"+ ).encode()+ request = urllib.request.Request(+ f"http://127.0.0.1:{httpd.server_port}/v1/audio/transcriptions",+ data=body,+ headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},+ method="POST",+ )++ with urllib.request.urlopen(request, timeout=10) as response:+ payload = json.loads(response.read().decode("utf-8"))++ assert response.status == 200+ assert payload == {"text": "hello from gguf"}+ args = json.loads(captured_args.read_text(encoding="utf-8"))+ assert args[:4] == ["-m", str(tmp_path / "sensevoice.gguf"), "-a", args[3]]+ assert os.path.exists(args[3]) is False+ assert args[4:] == ["--vad", str(tmp_path / "fsmn-vad.gguf"), "--ids"]+ finally:+ httpd.shutdown()+ httpd.server_close()diff --git a/runtime/llama.cpp/tests/test_release_workflow.py b/runtime/llama.cpp/tests/test_release_workflow.pyindex 201c97698..bddd44587 100644--- a/runtime/llama.cpp/tests/test_release_workflow.py+++ b/runtime/llama.cpp/tests/test_release_workflow.py@@ -41,3 +41,12 @@ def test_release_notes_explain_cpu_and_cuda_windows_assets():assert "other GPU architectures" in readmeassert "CMAKE_CUDA_ARCHITECTURES=120" in readmeassert "sm_120" in readme+++def test_readme_documents_lightweight_http_server():+ readme = (ROOT / "runtime" / "llama.cpp" / "README.md").read_text(encoding="utf-8")++ assert "server/funasr_gguf_server.py" in readme+ assert "/v1/audio/transcriptions" in readme+ assert "--binary ./build/bin/llama-funasr-sensevoice" in readme+ assert "one subprocess per request" in readme