Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat: q8_0 export option (--wtype q8_0) for SenseVoice/Paraformer/Nano encoder GGUF (#3013)

zhifu gao9f1d2204324b
diff --git a/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py b/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py
index 2804200f1..0ce6229ec 100644
--- a/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py
+++ b/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py
@@ -16,7 +16,7 @@ def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model_pt", required=True)
ap.add_argument("--out", required=True)
- ap.add_argument("--wtype", default="f32", choices=["f32", "f16"],
+ ap.add_argument("--wtype", default="f32", choices=["f32", "f16", "q8_0"],
help="dtype for 2D Linear (matmul) weights; norms/bias/fsmn stay f32")
args = ap.parse_args()
@@ -57,7 +57,11 @@ def main():
# matmul (Linear) weights -> optional f16; norms/biases/fsmn stay f32
elif args.wtype == "f16" and arr.ndim == 2 and "norm" not in k:
arr = arr.astype(np.float16)
- w.add_tensor(k, arr)
+ if args.wtype == "q8_0" and arr.ndim == 2 and "norm" not in k and "fsmn_block" not in k and arr.shape[1] % 32 == 0:
+ from gguf import quants as _q, GGMLQuantizationType as _QT
+ w.add_tensor(k, _q.quantize(arr, _QT.Q8_0), raw_dtype=_QT.Q8_0)
+ else:
+ w.add_tensor(k, arr)
n += 1
print(f"writing {n} tensors to {args.out}")
diff --git a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py
index 8a61aace2..0f607a396 100644
--- a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py
+++ b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py
@@ -19,7 +19,7 @@ def parse_mvn(path):
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model_pt", required=True); ap.add_argument("--mvn", required=True)
- ap.add_argument("--out", required=True); ap.add_argument("--wtype", default="f32", choices=["f32","f16"])
+ ap.add_argument("--out", required=True); ap.add_argument("--wtype", default="f32", choices=["f32","f16","q8_0"])
ap.add_argument("--tokens", default=None, help="tokens.json (id->token); default: next to model_pt")
a = ap.parse_args()
sd = torch.load(a.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd)
@@ -52,7 +52,12 @@ def main():
arr = np.ascontiguousarray(arr[:, 0, :].T)
elif args_f16(a) and arr.ndim == 2 and "norm" not in k and "cif_output" not in k:
arr = arr.astype(np.float16)
- w.add_tensor(k, arr); n += 1
+ if a.wtype == "q8_0" and arr.ndim == 2 and "norm" not in k and "fsmn_block" not in k and "predictor" not in k and arr.shape[1] % 32 == 0:
+ from gguf import quants as _q, GGMLQuantizationType as _QT
+ w.add_tensor(k, _q.quantize(arr, _QT.Q8_0), raw_dtype=_QT.Q8_0)
+ else:
+ w.add_tensor(k, arr)
+ n += 1
print(f"writing {n} tensors (+cmvn) to {a.out}")
w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()
print(f"done: {a.out} ({os.path.getsize(a.out)/1e6:.1f} MB)")
diff --git a/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py b/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py
index 57fda7e8b..17f88cf85 100644
--- a/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py
+++ b/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py
@@ -22,7 +22,7 @@ def main():
ap.add_argument("--model_pt", required=True)
ap.add_argument("--mvn", required=True)
ap.add_argument("--out", required=True)
- ap.add_argument("--wtype", default="f32", choices=["f32", "f16"])
+ ap.add_argument("--wtype", default="f32", choices=["f32", "f16", "q8_0"])
ap.add_argument("--spm", default=None, help="sentencepiece .bpe.model; default: next to model_pt")
args = ap.parse_args()
@@ -62,7 +62,11 @@ def main():
arr = np.ascontiguousarray(arr[:, 0, :].T)
elif args.wtype == "f16" and arr.ndim == 2 and "norm" not in k:
arr = arr.astype(np.float16)
- w.add_tensor(k, arr)
+ if args.wtype == "q8_0" and arr.ndim == 2 and "norm" not in k and "fsmn_block" not in k and k != "embed.weight" and arr.shape[1] % 32 == 0:
+ from gguf import quants as _q, GGMLQuantizationType as _QT
+ w.add_tensor(k, _q.quantize(arr, _QT.Q8_0), raw_dtype=_QT.Q8_0)
+ else:
+ w.add_tensor(k, arr)
n += 1
print(f"writing {n} tensors (+cmvn) to {args.out}")
w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()