Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(onnx): export at batch 2 with explicit Dims so the graph runs at any batch size
cklxx0e88902b9c62
diff --git a/scripts/export_onnx.py b/scripts/export_onnx.pyindex d71e3b02..7d0cf572 100644--- a/scripts/export_onnx.py+++ b/scripts/export_onnx.py@@ -50,18 +50,17 @@ def export_to_onnx(model_id_or_path: str, output_path: str):agent = Agent(model_id_or_path, compile=False, device="cpu")print("Creating dummy input tensors...")- # 1. Dummy tensors for tracing- # (batch_size=1, seq_len=16)- dummy_input_ids = torch.randint(0, 100, (1, 16), dtype=torch.long)- dummy_attention_mask = torch.ones((1, 16), dtype=torch.long)-- # (batch_size=1, num_markers=2)- dummy_marker_pos = torch.tensor([[1, 5]], dtype=torch.long)- dummy_marker_mask = torch.tensor([[True, True]], dtype=torch.bool)-- # (batch_size=1)- dummy_qtype = torch.tensor([0], dtype=torch.long)-+ # 1. Dummy tensors for tracing. torch.export specialises any dimension that is 1 (or equal+ # to another) at trace time, so a batch-1 dummy baked batch=1 into the decision head's+ # attention: the export ran at batch 1 and failed at batch >= 2 (#695). Batch, sequence and+ # marker counts are therefore all > 1 and pairwise different. laya-ts's exporter does the same.+ batch, seq_len, num_markers = 2, 17, 3+ dummy_input_ids = torch.randint(0, 100, (batch, seq_len), dtype=torch.long)+ dummy_attention_mask = torch.ones((batch, seq_len), dtype=torch.long)+ dummy_marker_pos = torch.tensor([[1, 5, 9]] * batch, dtype=torch.long)+ dummy_marker_mask = torch.ones((batch, num_markers), dtype=torch.bool)+ dummy_qtype = torch.zeros(batch, dtype=torch.long)+inputs = (dummy_input_ids,dummy_attention_mask,@@ -70,16 +69,19 @@ def export_to_onnx(model_id_or_path: str, output_path: str):dummy_qtype,)- # 2. Define dynamic axes so the model can accept variable batch sizes and sequence lengths- dynamic_axes = {- "input_ids": {0: "batch_size", 1: "seq_len"},- "attention_mask": {0: "batch_size", 1: "seq_len"},- "marker_pos": {0: "batch_size", 1: "num_markers"},- "marker_mask": {0: "batch_size", 1: "num_markers"},- "qtype": {0: "batch_size"},- "logits": {0: "batch_size", 1: "num_markers"},- "act_logits": {0: "batch_size"},- }+ # 2. Dynamic dimensions, declared as torch.export Dims (what the dynamo exporter reads;+ # `dynamic_axes` is only converted to these with a deprecation warning). The outputs follow+ # from the inputs, so `act_logits` is (batch_size, 2) instead of a static (1, 2).+ batch_dim = torch.export.Dim("batch_size")+ seq_dim = torch.export.Dim("seq_len")+ marker_dim = torch.export.Dim("num_markers")+ dynamic_shapes = (+ {0: batch_dim, 1: seq_dim},+ {0: batch_dim, 1: seq_dim},+ {0: batch_dim, 1: marker_dim},+ {0: batch_dim, 1: marker_dim},+ {0: batch_dim},+ )input_names = ["input_ids",@@ -110,7 +112,7 @@ def export_to_onnx(model_id_or_path: str, output_path: str):do_constant_folding=True,input_names=input_names,output_names=output_names,- dynamic_axes=dynamic_axes,+ dynamic_shapes=dynamic_shapes,)print(f"Successfully exported ONNX model to: {output_path}")diff --git a/tests/test_onnx.py b/tests/test_onnx.pyindex 5f35b59f..40fc9162 100644--- a/tests/test_onnx.py+++ b/tests/test_onnx.py@@ -2,6 +2,7 @@ import osimport sysimport tempfileimport numpy as np+import torchsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))@@ -88,4 +89,35 @@ def test_onnx_numerical_parity():atol=1e-3, rtol=1e-3,)+ # 8. The act head: its output was a static (1, 2) when the export traced at batch 1+ # (#695), and nothing above reads it.+ for qid in questions:+ np.testing.assert_allclose(+ res_pt["answers"][qid]["action"]["act_probability"],+ res_onnx["answers"][qid]["action"]["act_probability"],+ atol=5e-3,+ )++ # 9. The graph itself at batch 1 and 3 with a padded row, against the PyTorch model.+ # ONNXAgent collates one row per question, so the predict() above already ran at batch 3;+ # this pins the shapes and a padded key mask without any laya collation in the loop.+ for batch in (1, 3):+ rng = np.random.default_rng(batch)+ seq_len = 53+ feed = {+ "input_ids": rng.integers(5, 1000, (batch, seq_len)),+ "attention_mask": np.ones((batch, seq_len), dtype=np.int64),+ "marker_pos": np.tile(np.array([[3, 9, 14, 20, 30]]), (batch, 1)),+ "marker_mask": np.ones((batch, 5), dtype=bool),+ "qtype": np.arange(batch, dtype=np.int64) % 3,+ }+ feed["attention_mask"][0, 40:] = 0+ logits, act_logits = agent_onnx.session.run(["logits", "act_logits"], feed)+ assert logits.shape == (batch, 5) and act_logits.shape == (batch, 2), (logits.shape, act_logits.shape)+ with torch.no_grad():+ ref = agent_pt.model(*[torch.from_numpy(feed[n]) for n in+ ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")])+ np.testing.assert_allclose(logits, ref[0].numpy(), atol=1e-3)+ np.testing.assert_allclose(act_logits, ref[1].numpy(), atol=1e-2)+print("ONNX numerical parity test passed for all 3 question types!")