Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(onnx): default --quantize to per-tensor; per-channel collapses the model (#790)
AlKor13e6fe1884b53e
diff --git a/scripts/export_onnx.py b/scripts/export_onnx.pyindex 7d0cf572..28677711 100644--- a/scripts/export_onnx.py+++ b/scripts/export_onnx.py@@ -5,15 +5,28 @@ import torchfrom laya.agent import Agent-def quantize_model(model_path: str, output_path: str) -> str:+def quantize_model(model_path: str, output_path: str, per_channel: bool = False) -> str:"""Write an INT8 weight-only dynamically quantized copy of `model_path`.Dynamic quantization converts the weights of every `MatMul` (the attention and MLP linear- layers) to int8 while leaving activations in fp32; the quantization scales are computed per- output channel at load time, so no calibration dataset is needed. The graph structure and- the input/output names are unchanged, which is what lets `ONNXAgent` load the result by- pointing `onnx_path` at it. It is CPU-only: ONNX Runtime has no INT8 MatMul kernel on the- CUDAExecutionProvider, so an int8 graph on GPU falls back to CPU.+ layers) to int8 while leaving activations in fp32; the activation scale is computed per input+ at run time, so no calibration dataset is needed. The graph structure and the input/output+ names are unchanged, which is what lets `ONNXAgent` load the result by pointing `onnx_path` at+ it. It is CPU-only: ONNX Runtime has no INT8 MatMul kernel on the CUDAExecutionProvider, so an+ int8 graph on GPU falls back to CPU.++ `per_channel` defaults to **False** (one scale per weight tensor). Per-channel scales are a+ static/QDQ feature; on the dynamic `MatMulInteger` path the per-channel weight scale does not+ combine correctly with the per-input activation scale, and on the real checkpoints it collapses+ the decision model (measured: the English ModernBERT-large agreed with the eager Agent on only+ 31/96 held-out choice decisions at `per_channel=True` vs 64/96 at `per_channel=False`; the+ multilingual mmBERT was 40% vs 83%). See issue #790.++ INT8 is a size/latency option, not a free one. On CPU it is roughly 2x faster than eager and+ ~1.8x faster than the fp32 ONNX graph, and 1.4-2.8x smaller, but even at `per_channel=False` it+ trades real accuracy (the drift above is substantial, and worse on the larger checkpoint): do+ not use it where the calibrated probability or confidence matters. Accuracy-safe int8 for this+ model would need QAT or SmoothQuant-style outlier handling, not an export-time flag."""from onnxruntime.quantization import QuantType, quantize_dynamic@@ -30,11 +43,7 @@ def quantize_model(model_path: str, output_path: str) -> str:model_output=output_path,op_types_to_quantize=["MatMul"],weight_type=QuantType.QInt8,- # One scale per output channel rather than one per tensor. On the English checkpoint- # measured on 20 support-ticket states x choice/noul/score, per-tensor int8 flipped 3- # of 20 decisions (max probability drift 0.29); per-channel flipped none (max 0.09)- # at the same size and speed.- per_channel=True,+ per_channel=per_channel,)return output_path@@ -123,10 +132,16 @@ if __name__ == "__main__":parser.add_argument("--output", type=str, default="laya.onnx", help="Output path for the ONNX file")parser.add_argument("--quantize", action="store_true",help="Also write an INT8 weight-only quantized copy (CPU-only speed and "- "size win) next to --output, named <output>.int8.onnx")+ "size win, at a real accuracy cost) next to --output, named "+ "<output>.int8.onnx")+ parser.add_argument("--per-channel", action="store_true",+ help="Quantize weights per output channel instead of per tensor. Off by "+ "default: on the dynamic path per-channel collapses the model (see "+ "issue #790). Only meaningful with --quantize.")args = parser.parse_args()export_to_onnx(args.model, args.output)if args.quantize:- int8_path = quantize_model(args.output, int8_output_path(args.output))+ int8_path = quantize_model(args.output, int8_output_path(args.output),+ per_channel=args.per_channel)print(f"Successfully wrote INT8 quantized model to: {int8_path}")diff --git a/tests/test_onnx_quantize.py b/tests/test_onnx_quantize.pyindex 67fd3a2c..7097c7a9 100644--- a/tests/test_onnx_quantize.py+++ b/tests/test_onnx_quantize.py@@ -1,8 +1,8 @@"""INT8 export: `scripts/export_onnx.py --quantize` writes a weight-only quantized copy.`ONNXAgent` loads any graph that keeps the fp32 export's input/output names, so the whole-feature is one function: `quantize_model` (dynamic per-channel INT8 MatMul weights via-onnxruntime) plus+feature is one function: `quantize_model` (dynamic per-tensor INT8 MatMul weights via+onnxruntime; per-channel is an opt-in) plus`int8_output_path` (the sidecar naming) and the CLI flag. No checkpoint and no torch model isdownloaded here: the quantizer is exercised on a tiny hand-built MatMul graph, which is exactlythe shape it targets in the real export, and the CLI is checked through `--help`.@@ -87,9 +87,19 @@ int8_tensors = [t for t in q_model.graph.initializer if t.data_type == TensorProcheck_true("quantize/MatMul weight is stored as INT8", len(int8_tensors) >= 1,[(t.name, t.data_type) for t in q_model.graph.initializer])scales = [t for t in q_model.graph.initializer if t.name.endswith("_scale")]-check_true("quantize/scales are per output channel (per-tensor flips decisions on the real model)",- scales and all(numpy_helper.to_array(t).size > 1 for t in scales),+# Default is per-tensor (one scale per weight). Per-channel is off by default because on the+# dynamic MatMulInteger path it collapses the real decision model -- see issue #790.+check_true("quantize/default scales are per-tensor",+ scales and all(numpy_helper.to_array(t).size == 1 for t in scales),[(t.name, list(t.dims)) for t in scales])++# the opt-in still produces per-channel scales (one per output channel of the 8x8 weight)+pc_path = os.path.join(tmp, "mini.pc.int8.onnx")+export_onnx.quantize_model(fp32_path, pc_path, per_channel=True)+pc_scales = [t for t in onnx.load(pc_path).graph.initializer if t.name.endswith("_scale")]+check_true("quantize/per_channel=True opts into per-channel scales",+ pc_scales and any(numpy_helper.to_array(t).size > 1 for t in pc_scales),+ [(t.name, list(t.dims)) for t in pc_scales])check("quantize/graph keeps the I/O names ONNXAgent binds to",([i.name for i in q_model.graph.input], [o.name for o in q_model.graph.output]),(["input_ids"], ["logits"]))@@ -127,6 +137,8 @@ help_text = subprocess.run().stdoutcheck_true("cli/--quantize is documented in --help", "--quantize" in help_text,[ln for ln in help_text.splitlines() if "quantize" in ln])+check_true("cli/--per-channel opt-in is documented in --help", "--per-channel" in help_text,+ [ln for ln in help_text.splitlines() if "per-channel" in ln])check_true("cli/help names the sidecar output", ".int8.onnx" in help_text)