Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat: q8_0 export option (--wtype q8_0) for SenseVoice/Paraformer/Nano encoder GGUF (#3013)
zhifu gao9f1d2204324b
diff --git a/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py b/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.pyindex 2804200f1..0ce6229ec 100644--- a/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py+++ b/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py@@ -16,7 +16,7 @@ def main():ap = argparse.ArgumentParser()ap.add_argument("--model_pt", required=True)ap.add_argument("--out", required=True)- ap.add_argument("--wtype", default="f32", choices=["f32", "f16"],+ ap.add_argument("--wtype", default="f32", choices=["f32", "f16", "q8_0"],help="dtype for 2D Linear (matmul) weights; norms/bias/fsmn stay f32")args = ap.parse_args()@@ -57,7 +57,11 @@ def main():# matmul (Linear) weights -> optional f16; norms/biases/fsmn stay f32elif args.wtype == "f16" and arr.ndim == 2 and "norm" not in k:arr = arr.astype(np.float16)- w.add_tensor(k, arr)+ if args.wtype == "q8_0" and arr.ndim == 2 and "norm" not in k and "fsmn_block" not in k and arr.shape[1] % 32 == 0:+ from gguf import quants as _q, GGMLQuantizationType as _QT+ w.add_tensor(k, _q.quantize(arr, _QT.Q8_0), raw_dtype=_QT.Q8_0)+ else:+ w.add_tensor(k, arr)n += 1print(f"writing {n} tensors to {args.out}")diff --git a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py b/runtime/llama.cpp/paraformer/export_paraformer_gguf.pyindex 8a61aace2..0f607a396 100644--- a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py+++ b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py@@ -19,7 +19,7 @@ def parse_mvn(path):def main():ap = argparse.ArgumentParser()ap.add_argument("--model_pt", required=True); ap.add_argument("--mvn", required=True)- ap.add_argument("--out", required=True); ap.add_argument("--wtype", default="f32", choices=["f32","f16"])+ ap.add_argument("--out", required=True); ap.add_argument("--wtype", default="f32", choices=["f32","f16","q8_0"])ap.add_argument("--tokens", default=None, help="tokens.json (id->token); default: next to model_pt")a = ap.parse_args()sd = torch.load(a.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd)@@ -52,7 +52,12 @@ def main():arr = np.ascontiguousarray(arr[:, 0, :].T)elif args_f16(a) and arr.ndim == 2 and "norm" not in k and "cif_output" not in k:arr = arr.astype(np.float16)- w.add_tensor(k, arr); n += 1+ if a.wtype == "q8_0" and arr.ndim == 2 and "norm" not in k and "fsmn_block" not in k and "predictor" not in k and arr.shape[1] % 32 == 0:+ from gguf import quants as _q, GGMLQuantizationType as _QT+ w.add_tensor(k, _q.quantize(arr, _QT.Q8_0), raw_dtype=_QT.Q8_0)+ else:+ w.add_tensor(k, arr)+ n += 1print(f"writing {n} tensors (+cmvn) to {a.out}")w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()print(f"done: {a.out} ({os.path.getsize(a.out)/1e6:.1f} MB)")diff --git a/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py b/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.pyindex 57fda7e8b..17f88cf85 100644--- a/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py+++ b/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py@@ -22,7 +22,7 @@ def main():ap.add_argument("--model_pt", required=True)ap.add_argument("--mvn", required=True)ap.add_argument("--out", required=True)- ap.add_argument("--wtype", default="f32", choices=["f32", "f16"])+ ap.add_argument("--wtype", default="f32", choices=["f32", "f16", "q8_0"])ap.add_argument("--spm", default=None, help="sentencepiece .bpe.model; default: next to model_pt")args = ap.parse_args()@@ -62,7 +62,11 @@ def main():arr = np.ascontiguousarray(arr[:, 0, :].T)elif args.wtype == "f16" and arr.ndim == 2 and "norm" not in k:arr = arr.astype(np.float16)- w.add_tensor(k, arr)+ if args.wtype == "q8_0" and arr.ndim == 2 and "norm" not in k and "fsmn_block" not in k and k != "embed.weight" and arr.shape[1] % 32 == 0:+ from gguf import quants as _q, GGMLQuantizationType as _QT+ w.add_tensor(k, _q.quantize(arr, _QT.Q8_0), raw_dtype=_QT.Q8_0)+ else:+ w.add_tensor(k, arr)n += 1print(f"writing {n} tensors (+cmvn) to {args.out}")w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()