Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat(runtime): add GGUF HTTP transcription wrapper

zhifu gaob798e2679f9b
diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.md
index f38460507..6f7cd62e2 100644
--- a/runtime/llama.cpp/README.md
+++ b/runtime/llama.cpp/README.md
@@ -107,6 +107,43 @@ The shared **FSMN-VAD** front end builds the same way (`funasr-vad/` + `funasr-c
target `llama-funasr-vad`); export weights with `export_vad_gguf.py`. Pass
`--vad fsmn-vad.gguf` to any of the three tools for built-in long-audio segmentation.
+## Lightweight HTTP server
+
+The GGUF binaries are command-line tools first. For local apps that expect an
+HTTP transcription endpoint, `server/funasr_gguf_server.py` wraps an existing
+binary and exposes an OpenAI-compatible `POST /v1/audio/transcriptions` route.
+It uses only the Python standard library and still runs inference in the C++
+binary:
+
+```bash
+python server/funasr_gguf_server.py \
+ --host 127.0.0.1 --port 8000 \
+ --binary ./build/bin/llama-funasr-sensevoice \
+ --model ./gguf/sensevoice-small-q8.gguf \
+ --vad ./gguf/fsmn-vad.gguf
+```
+
+Then send audio with the same shape used by OpenAI-compatible clients:
+
+```bash
+curl http://127.0.0.1:8000/v1/audio/transcriptions \
+ -F model=funasr-gguf
+```
+
+Response:
+
+```json
+{"text": "transcribed text"}
+```
+
+CUDA-enabled SenseVoice builds can be selected with `--backend cuda`. Extra
+binary flags can be forwarded with repeated `--extra-arg`, for example
+`--extra-arg --keep-tags`. This wrapper starts one subprocess per request, so it
+is best for local tools, demos, and integration tests. For sustained production
+traffic, use the Python `funasr-server` OpenAI-compatible service or build a
+dedicated native server around the C++ runtime.
+
## Validation
Each model was validated against the FunASR PyTorch reference (encoder cosine ≈ 1.0;
diff --git a/runtime/llama.cpp/server/funasr_gguf_server.py b/runtime/llama.cpp/server/funasr_gguf_server.py
new file mode 100644
index 000000000..6c6463cf1
--- /dev/null
+++ b/runtime/llama.cpp/server/funasr_gguf_server.py
@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""OpenAI-compatible HTTP wrapper for FunASR llama.cpp / GGUF binaries.
+
+This server intentionally keeps inference in the existing C++ command-line
+tools. It accepts a multipart audio upload, runs the configured GGUF binary, and
+returns the transcript as a small OpenAI-compatible JSON response.
+"""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass, field
+from email import policy
+from email.parser import BytesParser
+import json
+from http import HTTPStatus
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+from typing import Iterable, Optional
+from urllib.parse import urlparse
+
+
+@dataclass(frozen=True)
+class ServerConfig:
+ binary: str
+ model: str
+ vad: Optional[str] = None
+ backend: Optional[str] = None
+ extra_args: list[str] = field(default_factory=list)
+ work_dir: str = field(default_factory=tempfile.gettempdir)
+ timeout: float = 600.0
+
+
+def build_command(config: ServerConfig, audio_path: str) -> list[str]:
+ command = [config.binary, "-m", config.model, "-a", audio_path]
+ if config.vad:
+ command.extend(["--vad", config.vad])
+ if config.backend:
+ command.extend(["--backend", config.backend])
+ command.extend(config.extra_args)
+ return command
+
+
+def extract_transcript(stdout: str) -> str:
+ return stdout.strip()
+
+
+def transcribe_file(config: ServerConfig, audio_path: str) -> str:
+ completed = subprocess.run(
+ build_command(config, audio_path),
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=config.timeout,
+ )
+ return extract_transcript(completed.stdout)
+
+
+def _json_bytes(payload: dict) -> bytes:
+ return json.dumps(payload, ensure_ascii=False).encode("utf-8")
+
+
+def _error_payload(message: str) -> bytes:
+ return _json_bytes({"error": {"message": message}})
+
+
+def _field_content_disposition(part) -> str:
+ return part.get("Content-Disposition", "")
+
+
+def _is_file_field(part) -> bool:
+ disposition = _field_content_disposition(part)
+ return "form-data" in disposition and 'name="file"' in disposition
+
+
+def parse_multipart_file(content_type: str, body: bytes) -> tuple[bytes, str]:
+ message = BytesParser(policy=policy.default).parsebytes(
+ (
+ f"Content-Type: {content_type}\r\n"
+ "MIME-Version: 1.0\r\n"
+ "\r\n"
+ ).encode("utf-8")
+ + body
+ )
+ if not message.is_multipart():
+ raise ValueError("expected multipart/form-data")
+
+ for part in message.iter_parts():
+ if not _is_file_field(part):
+ continue
+ filename = part.get_filename() or "audio.wav"
+ payload = part.get_payload(decode=True) or b""
+ if not payload:
+ raise ValueError("uploaded file is empty")
+ return payload, filename
+ raise ValueError("missing multipart field: file")
+
+
+def _suffix_from_filename(filename: str) -> str:
+ suffix = Path(filename).suffix
+ if not suffix or len(suffix) > 16:
+ return ".wav"
+ return suffix
+
+
+class FunASRGGUFHandler(BaseHTTPRequestHandler):
+ server_version = "FunASRGGUFServer/0.1"
+
+ def log_message(self, fmt: str, *args) -> None:
+ print("%s - - [%s] %s" % (self.address_string(), self.log_date_time_string(), fmt % args))
+
+ @property
+ def config(self) -> ServerConfig:
+ return self.server.config # type: ignore[attr-defined]
+
+ def _send_json(self, status: HTTPStatus, payload: bytes) -> None:
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json; charset=utf-8")
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+ def do_GET(self) -> None:
+ if urlparse(self.path).path == "/health":
+ self._send_json(HTTPStatus.OK, _json_bytes({"status": "ok"}))
+ return
+ self._send_json(HTTPStatus.NOT_FOUND, _error_payload("not found"))
+
+ def do_POST(self) -> None:
+ if urlparse(self.path).path != "/v1/audio/transcriptions":
+ self._send_json(HTTPStatus.NOT_FOUND, _error_payload("not found"))
+ return
+
+ try:
+ length = int(self.headers.get("Content-Length", "0"))
+ content_type = self.headers.get("Content-Type", "")
+ audio_bytes, filename = parse_multipart_file(content_type, self.rfile.read(length))
+ except Exception as exc:
+ self._send_json(HTTPStatus.BAD_REQUEST, _error_payload(str(exc)))
+ return
+
+ tmp_path = None
+ try:
+ os.makedirs(self.config.work_dir, exist_ok=True)
+ with tempfile.NamedTemporaryFile(
+ "wb",
+ suffix=_suffix_from_filename(filename),
+ dir=self.config.work_dir,
+ delete=False,
+ ) as tmp:
+ tmp.write(audio_bytes)
+ tmp_path = tmp.name
+ transcript = transcribe_file(self.config, tmp_path)
+ except subprocess.CalledProcessError as exc:
+ message = exc.stderr.strip() or exc.stdout.strip() or str(exc)
+ self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, _error_payload(message))
+ return
+ except subprocess.TimeoutExpired:
+ self._send_json(HTTPStatus.GATEWAY_TIMEOUT, _error_payload("transcription timed out"))
+ return
+ except Exception as exc:
+ self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, _error_payload(str(exc)))
+ return
+ finally:
+ if tmp_path:
+ try:
+ os.unlink(tmp_path)
+ except FileNotFoundError:
+ pass
+
+ self._send_json(HTTPStatus.OK, _json_bytes({"text": transcript}))
+
+
+class FunASRHTTPServer(ThreadingHTTPServer):
+ def __init__(self, server_address, handler_class, config: ServerConfig):
+ super().__init__(server_address, handler_class)
+ self.config = config
+
+
+def create_server(host: str, port: int, config: ServerConfig) -> FunASRHTTPServer:
+ return FunASRHTTPServer((host, port), FunASRGGUFHandler, config)
+
+
+def _parse_extra_args(values: Optional[Iterable[str]]) -> list[str]:
+ args: list[str] = []
+ for value in values or []:
+ args.extend(value.split())
+ return args
+
+
+def build_arg_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="Serve a FunASR llama.cpp / GGUF binary over HTTP.")
+ parser.add_argument("--host", default="127.0.0.1")
+ parser.add_argument("--port", type=int, default=8000)
+ parser.add_argument("--binary", required=True, help="Path to llama-funasr-sensevoice or llama-funasr-paraformer.")
+ parser.add_argument("--model", required=True, help="Path to the model GGUF passed as -m.")
+ parser.add_argument("--vad", help="Optional FSMN-VAD GGUF passed as --vad.")
+ parser.add_argument("--backend", choices=["cpu", "cuda"], help="Optional backend passed as --backend.")
+ parser.add_argument("--work-dir", default=tempfile.gettempdir(), help="Directory for temporary uploaded audio files.")
+ parser.add_argument("--timeout", type=float, default=600.0, help="Per-request subprocess timeout in seconds.")
+ parser.add_argument(
+ "--extra-arg",
+ action="append",
+ default=[],
+ help="Additional argument(s) forwarded to the GGUF binary. May be repeated.",
+ )
+ return parser
+
+
+def main() -> None:
+ args = build_arg_parser().parse_args()
+ config = ServerConfig(
+ binary=args.binary,
+ model=args.model,
+ vad=args.vad,
+ backend=args.backend,
+ extra_args=_parse_extra_args(args.extra_arg),
+ work_dir=args.work_dir,
+ timeout=args.timeout,
+ )
+ httpd = create_server(args.host, args.port, config)
+ print(f"Serving FunASR GGUF transcription on http://{args.host}:{httpd.server_port}")
+ httpd.serve_forever()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/runtime/llama.cpp/tests/test_gguf_server.py b/runtime/llama.cpp/tests/test_gguf_server.py
new file mode 100644
index 000000000..f24a5cf12
--- /dev/null
+++ b/runtime/llama.cpp/tests/test_gguf_server.py
@@ -0,0 +1,100 @@
+import importlib.util
+import json
+import os
+import sys
+import threading
+import urllib.request
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SERVER = ROOT / "server" / "funasr_gguf_server.py"
+
+
+def load_server_module():
+ spec = importlib.util.spec_from_file_location("funasr_gguf_server", SERVER)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules["funasr_gguf_server"] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_build_command_adds_model_audio_vad_backend_and_extra_args(tmp_path):
+ server = load_server_module()
+ cfg = server.ServerConfig(
+ binary="/opt/funasr/llama-funasr-sensevoice",
+ model="/models/sensevoice.gguf",
+ vad="/models/fsmn-vad.gguf",
+ backend="cuda",
+ extra_args=["--keep-tags"],
+ work_dir=str(tmp_path),
+ )
+
+ command = server.build_command(cfg, "/tmp/request.wav")
+
+ assert command == [
+ "/opt/funasr/llama-funasr-sensevoice",
+ "-m",
+ "/models/sensevoice.gguf",
+ "-a",
+ "/tmp/request.wav",
+ "--vad",
+ "/models/fsmn-vad.gguf",
+ "--backend",
+ "cuda",
+ "--keep-tags",
+ ]
+
+
+def test_transcription_endpoint_runs_binary_and_returns_openai_json(tmp_path):
+ server = load_server_module()
+ fake_binary = tmp_path / "fake_funasr.py"
+ captured_args = tmp_path / "args.json"
+ fake_binary.write_text(
+ "#!/usr/bin/env python3\n"
+ "import json, pathlib, sys\n"
+ f"pathlib.Path({str(captured_args)!r}).write_text(json.dumps(sys.argv[1:]))\n"
+ "print('hello from gguf')\n",
+ encoding="utf-8",
+ )
+ fake_binary.chmod(0o755)
+
+ cfg = server.ServerConfig(
+ binary=str(fake_binary),
+ model=str(tmp_path / "sensevoice.gguf"),
+ vad=str(tmp_path / "fsmn-vad.gguf"),
+ backend=None,
+ extra_args=["--ids"],
+ work_dir=str(tmp_path),
+ )
+ httpd = server.create_server("127.0.0.1", 0, cfg)
+ thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+ thread.start()
+ try:
+ boundary = "----funasr-test-boundary"
+ body = (
+ f"--{boundary}\r\n"
+ 'Content-Disposition: form-data; name="file"; filename="sample.wav"\r\n'
+ "Content-Type: audio/wav\r\n\r\n"
+ "RIFFfake-audio\r\n"
+ f"--{boundary}--\r\n"
+ ).encode()
+ request = urllib.request.Request(
+ f"http://127.0.0.1:{httpd.server_port}/v1/audio/transcriptions",
+ data=body,
+ headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
+ method="POST",
+ )
+
+ with urllib.request.urlopen(request, timeout=10) as response:
+ payload = json.loads(response.read().decode("utf-8"))
+
+ assert response.status == 200
+ assert payload == {"text": "hello from gguf"}
+ args = json.loads(captured_args.read_text(encoding="utf-8"))
+ assert args[:4] == ["-m", str(tmp_path / "sensevoice.gguf"), "-a", args[3]]
+ assert os.path.exists(args[3]) is False
+ assert args[4:] == ["--vad", str(tmp_path / "fsmn-vad.gguf"), "--ids"]
+ finally:
+ httpd.shutdown()
+ httpd.server_close()
diff --git a/runtime/llama.cpp/tests/test_release_workflow.py b/runtime/llama.cpp/tests/test_release_workflow.py
index 201c97698..bddd44587 100644
--- a/runtime/llama.cpp/tests/test_release_workflow.py
+++ b/runtime/llama.cpp/tests/test_release_workflow.py
@@ -41,3 +41,12 @@ def test_release_notes_explain_cpu_and_cuda_windows_assets():
assert "other GPU architectures" in readme
assert "CMAKE_CUDA_ARCHITECTURES=120" in readme
assert "sm_120" in readme
+
+
+def test_readme_documents_lightweight_http_server():
+ readme = (ROOT / "runtime" / "llama.cpp" / "README.md").read_text(encoding="utf-8")
+
+ assert "server/funasr_gguf_server.py" in readme
+ assert "/v1/audio/transcriptions" in readme
+ assert "--binary ./build/bin/llama-funasr-sensevoice" in readme
+ assert "one subprocess per request" in readme