Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(onnx): default --quantize to per-tensor; per-channel collapses the model (#790)

AlKor13e6fe1884b53e
diff --git a/scripts/export_onnx.py b/scripts/export_onnx.py
index 7d0cf572..28677711 100644
--- a/scripts/export_onnx.py
+++ b/scripts/export_onnx.py
@@ -5,15 +5,28 @@ import torch
from laya.agent import Agent
-def quantize_model(model_path: str, output_path: str) -> str:
+def quantize_model(model_path: str, output_path: str, per_channel: bool = False) -> str:
"""Write an INT8 weight-only dynamically quantized copy of `model_path`.
Dynamic quantization converts the weights of every `MatMul` (the attention and MLP linear
- layers) to int8 while leaving activations in fp32; the quantization scales are computed per
- output channel at load time, so no calibration dataset is needed. The graph structure and
- the input/output names are unchanged, which is what lets `ONNXAgent` load the result by
- pointing `onnx_path` at it. It is CPU-only: ONNX Runtime has no INT8 MatMul kernel on the
- CUDAExecutionProvider, so an int8 graph on GPU falls back to CPU.
+ layers) to int8 while leaving activations in fp32; the activation scale is computed per input
+ at run time, so no calibration dataset is needed. The graph structure and the input/output
+ names are unchanged, which is what lets `ONNXAgent` load the result by pointing `onnx_path` at
+ it. It is CPU-only: ONNX Runtime has no INT8 MatMul kernel on the CUDAExecutionProvider, so an
+ int8 graph on GPU falls back to CPU.
+
+ `per_channel` defaults to **False** (one scale per weight tensor). Per-channel scales are a
+ static/QDQ feature; on the dynamic `MatMulInteger` path the per-channel weight scale does not
+ combine correctly with the per-input activation scale, and on the real checkpoints it collapses
+ the decision model (measured: the English ModernBERT-large agreed with the eager Agent on only
+ 31/96 held-out choice decisions at `per_channel=True` vs 64/96 at `per_channel=False`; the
+ multilingual mmBERT was 40% vs 83%). See issue #790.
+
+ INT8 is a size/latency option, not a free one. On CPU it is roughly 2x faster than eager and
+ ~1.8x faster than the fp32 ONNX graph, and 1.4-2.8x smaller, but even at `per_channel=False` it
+ trades real accuracy (the drift above is substantial, and worse on the larger checkpoint): do
+ not use it where the calibrated probability or confidence matters. Accuracy-safe int8 for this
+ model would need QAT or SmoothQuant-style outlier handling, not an export-time flag.
"""
from onnxruntime.quantization import QuantType, quantize_dynamic
@@ -30,11 +43,7 @@ def quantize_model(model_path: str, output_path: str) -> str:
model_output=output_path,
op_types_to_quantize=["MatMul"],
weight_type=QuantType.QInt8,
- # One scale per output channel rather than one per tensor. On the English checkpoint
- # measured on 20 support-ticket states x choice/noul/score, per-tensor int8 flipped 3
- # of 20 decisions (max probability drift 0.29); per-channel flipped none (max 0.09)
- # at the same size and speed.
- per_channel=True,
+ per_channel=per_channel,
)
return output_path
@@ -123,10 +132,16 @@ if __name__ == "__main__":
parser.add_argument("--output", type=str, default="laya.onnx", help="Output path for the ONNX file")
parser.add_argument("--quantize", action="store_true",
help="Also write an INT8 weight-only quantized copy (CPU-only speed and "
- "size win) next to --output, named <output>.int8.onnx")
+ "size win, at a real accuracy cost) next to --output, named "
+ "<output>.int8.onnx")
+ parser.add_argument("--per-channel", action="store_true",
+ help="Quantize weights per output channel instead of per tensor. Off by "
+ "default: on the dynamic path per-channel collapses the model (see "
+ "issue #790). Only meaningful with --quantize.")
args = parser.parse_args()
export_to_onnx(args.model, args.output)
if args.quantize:
- int8_path = quantize_model(args.output, int8_output_path(args.output))
+ int8_path = quantize_model(args.output, int8_output_path(args.output),
+ per_channel=args.per_channel)
print(f"Successfully wrote INT8 quantized model to: {int8_path}")
diff --git a/tests/test_onnx_quantize.py b/tests/test_onnx_quantize.py
index 67fd3a2c..7097c7a9 100644
--- a/tests/test_onnx_quantize.py
+++ b/tests/test_onnx_quantize.py
@@ -1,8 +1,8 @@
"""INT8 export: `scripts/export_onnx.py --quantize` writes a weight-only quantized copy.
`ONNXAgent` loads any graph that keeps the fp32 export's input/output names, so the whole
-feature is one function: `quantize_model` (dynamic per-channel INT8 MatMul weights via
-onnxruntime) plus
+feature is one function: `quantize_model` (dynamic per-tensor INT8 MatMul weights via
+onnxruntime; per-channel is an opt-in) plus
`int8_output_path` (the sidecar naming) and the CLI flag. No checkpoint and no torch model is
downloaded here: the quantizer is exercised on a tiny hand-built MatMul graph, which is exactly
the shape it targets in the real export, and the CLI is checked through `--help`.
@@ -87,9 +87,19 @@ int8_tensors = [t for t in q_model.graph.initializer if t.data_type == TensorPro
check_true("quantize/MatMul weight is stored as INT8", len(int8_tensors) >= 1,
[(t.name, t.data_type) for t in q_model.graph.initializer])
scales = [t for t in q_model.graph.initializer if t.name.endswith("_scale")]
-check_true("quantize/scales are per output channel (per-tensor flips decisions on the real model)",
- scales and all(numpy_helper.to_array(t).size > 1 for t in scales),
+# Default is per-tensor (one scale per weight). Per-channel is off by default because on the
+# dynamic MatMulInteger path it collapses the real decision model -- see issue #790.
+check_true("quantize/default scales are per-tensor",
+ scales and all(numpy_helper.to_array(t).size == 1 for t in scales),
[(t.name, list(t.dims)) for t in scales])
+
+# the opt-in still produces per-channel scales (one per output channel of the 8x8 weight)
+pc_path = os.path.join(tmp, "mini.pc.int8.onnx")
+export_onnx.quantize_model(fp32_path, pc_path, per_channel=True)
+pc_scales = [t for t in onnx.load(pc_path).graph.initializer if t.name.endswith("_scale")]
+check_true("quantize/per_channel=True opts into per-channel scales",
+ pc_scales and any(numpy_helper.to_array(t).size > 1 for t in pc_scales),
+ [(t.name, list(t.dims)) for t in pc_scales])
check("quantize/graph keeps the I/O names ONNXAgent binds to",
([i.name for i in q_model.graph.input], [o.name for o in q_model.graph.output]),
(["input_ids"], ["logits"]))
@@ -127,6 +137,8 @@ help_text = subprocess.run(
).stdout
check_true("cli/--quantize is documented in --help", "--quantize" in help_text,
[ln for ln in help_text.splitlines() if "quantize" in ln])
+check_true("cli/--per-channel opt-in is documented in --help", "--per-channel" in help_text,
+ [ln for ln in help_text.splitlines() if "per-channel" in ln])
check_true("cli/help names the sidecar output", ".int8.onnx" in help_text)