Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(onnx): declare dynamic dims with dynamic_axes again so export works on torch < 2.9
cklxxaec8f8cfc65b
diff --git a/scripts/export_onnx.py b/scripts/export_onnx.pyindex 7d0cf572..46b6c331 100644--- a/scripts/export_onnx.py+++ b/scripts/export_onnx.py@@ -69,19 +69,20 @@ def export_to_onnx(model_id_or_path: str, output_path: str):dummy_qtype,)- # 2. Dynamic dimensions, declared as torch.export Dims (what the dynamo exporter reads;- # `dynamic_axes` is only converted to these with a deprecation warning). The outputs follow- # from the inputs, so `act_logits` is (batch_size, 2) instead of a static (1, 2).- batch_dim = torch.export.Dim("batch_size")- seq_dim = torch.export.Dim("seq_len")- marker_dim = torch.export.Dim("num_markers")- dynamic_shapes = (- {0: batch_dim, 1: seq_dim},- {0: batch_dim, 1: seq_dim},- {0: batch_dim, 1: marker_dim},- {0: batch_dim, 1: marker_dim},- {0: batch_dim},- )+ # 2. Dynamic dimensions. `dynamic_axes` rather than torch.export Dims: it is the declaration+ # every torch this package supports accepts (`dynamic_shapes` raises under the TorchScript+ # exporter, the default through torch 2.8, and does not exist before 2.5), and the dynamo+ # exporter converts it to Dims. With the batch-2 dummies above, both give the same graph;+ # the dummies are what fix #695, not the declaration.+ dynamic_axes = {+ "input_ids": {0: "batch_size", 1: "seq_len"},+ "attention_mask": {0: "batch_size", 1: "seq_len"},+ "marker_pos": {0: "batch_size", 1: "num_markers"},+ "marker_mask": {0: "batch_size", 1: "num_markers"},+ "qtype": {0: "batch_size"},+ "logits": {0: "batch_size", 1: "num_markers"},+ "act_logits": {0: "batch_size"},+ }input_names = ["input_ids",@@ -112,7 +113,7 @@ def export_to_onnx(model_id_or_path: str, output_path: str):do_constant_folding=True,input_names=input_names,output_names=output_names,- dynamic_shapes=dynamic_shapes,+ dynamic_axes=dynamic_axes,)print(f"Successfully exported ONNX model to: {output_path}")diff --git a/tests/test_onnx.py b/tests/test_onnx.pyindex 40fc9162..6b2801b1 100644--- a/tests/test_onnx.py+++ b/tests/test_onnx.py@@ -19,105 +19,115 @@ except ImportError:import pytest[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")-def test_onnx_numerical_parity():- """Verify that PyTorch and ONNX agents produce identical outputs for all 3 question types."""- model_id = "convaiinnovations/laya"-+MODEL_ID = "convaiinnovations/laya"+INPUT_NAMES = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")++[email protected](scope="module")+def exported():+ """One export shared by the tests below: (onnx path, the PyTorch agent it came from)."""with tempfile.TemporaryDirectory() as tmpdir:onnx_path = os.path.join(tmpdir, "laya.onnx")-- # 1. Export to ONNX- export_to_onnx(model_id, onnx_path)-- # 2. Load both PyTorch and ONNX Agents- agent_pt = Agent(model_id, compile=False, device="cpu")- agent_onnx = ONNXAgent(model_id, onnx_path)-- # 3. Define a state and all 3 question types- state = {"request": "Refactor this service using dependency injection"}- questions = {- "intent": {- "type": "choice",- "instructions": "What is the user asking to do?",- "criteria": {- "refactor": "code refactoring, rewriting",- "bug_fix": "fixing bugs, issues",- "feature": "adding new features"- }- },- "complexity": {- "type": "score",- "instructions": "How complex is this request?",- "criteria": ["trivial", "simple", "moderate", "complex", "very complex"]- },- "safety": {- "type": "noul",- "instructions": "Does this request involve any unsafe or harmful content?"- },+ export_to_onnx(MODEL_ID, onnx_path)+ yield onnx_path, Agent(MODEL_ID, compile=False, device="cpu")++[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")+def test_onnx_graph_runs_at_any_batch(exported):+ """The exported graph itself at batch 1, 3 and 7 with a padded row, against the PyTorch model.++ Runs before any ONNXAgent call, so an export that only works at batch 1 (#695: the trace+ specialised batch=1 and `act_logits` came out as a static (1, 2)) fails here, on the shape.+ """+ onnx_path, agent_pt = exported+ session = onnxruntime.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])+ for batch in (1, 3, 7):+ rng = np.random.default_rng(batch)+ seq_len = 53+ feed = {+ "input_ids": rng.integers(5, 1000, (batch, seq_len)),+ "attention_mask": np.ones((batch, seq_len), dtype=np.int64),+ "marker_pos": np.tile(np.array([[3, 9, 14, 20, 30]]), (batch, 1)),+ "marker_mask": np.ones((batch, 5), dtype=bool),+ "qtype": np.arange(batch, dtype=np.int64) % 3,}-- # 4. Predict with both agents- res_pt = agent_pt.predict(state, questions)- res_onnx = agent_onnx.predict(state, questions)-- # 5. Assert parity for choice question- assert res_pt["answers"]["intent"]["choice"] == res_onnx["answers"]["intent"]["choice"], \- f"Choice mismatch: {res_pt['answers']['intent']['choice']} vs {res_onnx['answers']['intent']['choice']}"-- probs_pt = res_pt["answers"]["intent"]["probabilities"]- probs_onnx = res_onnx["answers"]["intent"]["probabilities"]- for k in probs_pt.keys():- np.testing.assert_allclose(probs_pt[k], probs_onnx[k], atol=1e-3, rtol=1e-3)-- # 6. Assert parity for score question- np.testing.assert_allclose(- res_pt["answers"]["complexity"]["score"],- res_onnx["answers"]["complexity"]["score"],- atol=1e-3, rtol=1e-3,- )-- probs_pt_s = res_pt["answers"]["complexity"]["probabilities"]- probs_onnx_s = res_onnx["answers"]["complexity"]["probabilities"]- for k in probs_pt_s.keys():- np.testing.assert_allclose(probs_pt_s[k], probs_onnx_s[k], atol=1e-3, rtol=1e-3)-- # 7. Assert parity for noul question- np.testing.assert_allclose(- res_pt["answers"]["safety"]["noul"],- res_onnx["answers"]["safety"]["noul"],- atol=1e-3, rtol=1e-3,- )-- # 8. The act head: its output was a static (1, 2) when the export traced at batch 1- # (#695), and nothing above reads it.- for qid in questions:- np.testing.assert_allclose(- res_pt["answers"][qid]["action"]["act_probability"],- res_onnx["answers"][qid]["action"]["act_probability"],- atol=5e-3,- )+ feed["attention_mask"][0, 40:] = 0+ logits, act_logits = session.run(["logits", "act_logits"], feed)+ assert logits.shape == (batch, 5) and act_logits.shape == (batch, 2), (logits.shape, act_logits.shape)+ with torch.no_grad():+ ref = agent_pt.model(*[torch.from_numpy(feed[n]) for n in INPUT_NAMES])+ np.testing.assert_allclose(logits, ref[0].numpy(), atol=1e-3)+ np.testing.assert_allclose(act_logits, ref[1].numpy(), atol=1e-2)- # 9. The graph itself at batch 1 and 3 with a padded row, against the PyTorch model.- # ONNXAgent collates one row per question, so the predict() above already ran at batch 3;- # this pins the shapes and a padded key mask without any laya collation in the loop.- for batch in (1, 3):- rng = np.random.default_rng(batch)- seq_len = 53- feed = {- "input_ids": rng.integers(5, 1000, (batch, seq_len)),- "attention_mask": np.ones((batch, seq_len), dtype=np.int64),- "marker_pos": np.tile(np.array([[3, 9, 14, 20, 30]]), (batch, 1)),- "marker_mask": np.ones((batch, 5), dtype=bool),- "qtype": np.arange(batch, dtype=np.int64) % 3,+[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")+def test_onnx_numerical_parity(exported):+ """Verify that PyTorch and ONNX agents produce identical outputs for all 3 question types."""+ onnx_path, agent_pt = exported+ agent_onnx = ONNXAgent(MODEL_ID, onnx_path)++ # 3. Define a state and all 3 question types+ state = {"request": "Refactor this service using dependency injection"}+ questions = {+ "intent": {+ "type": "choice",+ "instructions": "What is the user asking to do?",+ "criteria": {+ "refactor": "code refactoring, rewriting",+ "bug_fix": "fixing bugs, issues",+ "feature": "adding new features"}- feed["attention_mask"][0, 40:] = 0- logits, act_logits = agent_onnx.session.run(["logits", "act_logits"], feed)- assert logits.shape == (batch, 5) and act_logits.shape == (batch, 2), (logits.shape, act_logits.shape)- with torch.no_grad():- ref = agent_pt.model(*[torch.from_numpy(feed[n]) for n in- ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")])- np.testing.assert_allclose(logits, ref[0].numpy(), atol=1e-3)- np.testing.assert_allclose(act_logits, ref[1].numpy(), atol=1e-2)+ },+ "complexity": {+ "type": "score",+ "instructions": "How complex is this request?",+ "criteria": ["trivial", "simple", "moderate", "complex", "very complex"]+ },+ "safety": {+ "type": "noul",+ "instructions": "Does this request involve any unsafe or harmful content?"+ },+ }++ # 4. Predict with both agents+ res_pt = agent_pt.predict(state, questions)+ res_onnx = agent_onnx.predict(state, questions)++ # 5. Assert parity for choice question+ assert res_pt["answers"]["intent"]["choice"] == res_onnx["answers"]["intent"]["choice"], \+ f"Choice mismatch: {res_pt['answers']['intent']['choice']} vs {res_onnx['answers']['intent']['choice']}"++ probs_pt = res_pt["answers"]["intent"]["probabilities"]+ probs_onnx = res_onnx["answers"]["intent"]["probabilities"]+ for k in probs_pt.keys():+ np.testing.assert_allclose(probs_pt[k], probs_onnx[k], atol=1e-3, rtol=1e-3)++ # 6. Assert parity for score question+ np.testing.assert_allclose(+ res_pt["answers"]["complexity"]["score"],+ res_onnx["answers"]["complexity"]["score"],+ atol=1e-3, rtol=1e-3,+ )++ probs_pt_s = res_pt["answers"]["complexity"]["probabilities"]+ probs_onnx_s = res_onnx["answers"]["complexity"]["probabilities"]+ for k in probs_pt_s.keys():+ np.testing.assert_allclose(probs_pt_s[k], probs_onnx_s[k], atol=1e-3, rtol=1e-3)++ # 7. Assert parity for noul question+ np.testing.assert_allclose(+ res_pt["answers"]["safety"]["noul"],+ res_onnx["answers"]["safety"]["noul"],+ atol=1e-3, rtol=1e-3,+ )++ # 8. The act head: its output was a static (1, 2) when the export traced at batch 1+ # (#695), and nothing above reads it.+ for qid in questions:+ np.testing.assert_allclose(+ res_pt["answers"][qid]["action"]["act_probability"],+ res_onnx["answers"][qid]["action"]["act_probability"],+ atol=5e-3,+ )- print("ONNX numerical parity test passed for all 3 question types!")+ print("ONNX numerical parity test passed for all 3 question types!")