Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(onnx): declare dynamic dims with dynamic_axes again so export works on torch < 2.9

cklxxaec8f8cfc65b
diff --git a/scripts/export_onnx.py b/scripts/export_onnx.py
index 7d0cf572..46b6c331 100644
--- a/scripts/export_onnx.py
+++ b/scripts/export_onnx.py
@@ -69,19 +69,20 @@ def export_to_onnx(model_id_or_path: str, output_path: str):
dummy_qtype,
)
- # 2. Dynamic dimensions, declared as torch.export Dims (what the dynamo exporter reads;
- # `dynamic_axes` is only converted to these with a deprecation warning). The outputs follow
- # from the inputs, so `act_logits` is (batch_size, 2) instead of a static (1, 2).
- batch_dim = torch.export.Dim("batch_size")
- seq_dim = torch.export.Dim("seq_len")
- marker_dim = torch.export.Dim("num_markers")
- dynamic_shapes = (
- {0: batch_dim, 1: seq_dim},
- {0: batch_dim, 1: seq_dim},
- {0: batch_dim, 1: marker_dim},
- {0: batch_dim, 1: marker_dim},
- {0: batch_dim},
- )
+ # 2. Dynamic dimensions. `dynamic_axes` rather than torch.export Dims: it is the declaration
+ # every torch this package supports accepts (`dynamic_shapes` raises under the TorchScript
+ # exporter, the default through torch 2.8, and does not exist before 2.5), and the dynamo
+ # exporter converts it to Dims. With the batch-2 dummies above, both give the same graph;
+ # the dummies are what fix #695, not the declaration.
+ dynamic_axes = {
+ "input_ids": {0: "batch_size", 1: "seq_len"},
+ "attention_mask": {0: "batch_size", 1: "seq_len"},
+ "marker_pos": {0: "batch_size", 1: "num_markers"},
+ "marker_mask": {0: "batch_size", 1: "num_markers"},
+ "qtype": {0: "batch_size"},
+ "logits": {0: "batch_size", 1: "num_markers"},
+ "act_logits": {0: "batch_size"},
+ }
input_names = [
"input_ids",
@@ -112,7 +113,7 @@ def export_to_onnx(model_id_or_path: str, output_path: str):
do_constant_folding=True,
input_names=input_names,
output_names=output_names,
- dynamic_shapes=dynamic_shapes,
+ dynamic_axes=dynamic_axes,
)
print(f"Successfully exported ONNX model to: {output_path}")
diff --git a/tests/test_onnx.py b/tests/test_onnx.py
index 40fc9162..6b2801b1 100644
--- a/tests/test_onnx.py
+++ b/tests/test_onnx.py
@@ -19,105 +19,115 @@ except ImportError:
import pytest
[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")
-def test_onnx_numerical_parity():
- """Verify that PyTorch and ONNX agents produce identical outputs for all 3 question types."""
- model_id = "convaiinnovations/laya"
-
+MODEL_ID = "convaiinnovations/laya"
+INPUT_NAMES = ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")
+
+
[email protected](scope="module")
+def exported():
+ """One export shared by the tests below: (onnx path, the PyTorch agent it came from)."""
with tempfile.TemporaryDirectory() as tmpdir:
onnx_path = os.path.join(tmpdir, "laya.onnx")
-
- # 1. Export to ONNX
- export_to_onnx(model_id, onnx_path)
-
- # 2. Load both PyTorch and ONNX Agents
- agent_pt = Agent(model_id, compile=False, device="cpu")
- agent_onnx = ONNXAgent(model_id, onnx_path)
-
- # 3. Define a state and all 3 question types
- state = {"request": "Refactor this service using dependency injection"}
- questions = {
- "intent": {
- "type": "choice",
- "instructions": "What is the user asking to do?",
- "criteria": {
- "refactor": "code refactoring, rewriting",
- "bug_fix": "fixing bugs, issues",
- "feature": "adding new features"
- }
- },
- "complexity": {
- "type": "score",
- "instructions": "How complex is this request?",
- "criteria": ["trivial", "simple", "moderate", "complex", "very complex"]
- },
- "safety": {
- "type": "noul",
- "instructions": "Does this request involve any unsafe or harmful content?"
- },
+ export_to_onnx(MODEL_ID, onnx_path)
+ yield onnx_path, Agent(MODEL_ID, compile=False, device="cpu")
+
+
[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")
+def test_onnx_graph_runs_at_any_batch(exported):
+ """The exported graph itself at batch 1, 3 and 7 with a padded row, against the PyTorch model.
+
+ Runs before any ONNXAgent call, so an export that only works at batch 1 (#695: the trace
+ specialised batch=1 and `act_logits` came out as a static (1, 2)) fails here, on the shape.
+ """
+ onnx_path, agent_pt = exported
+ session = onnxruntime.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])
+ for batch in (1, 3, 7):
+ rng = np.random.default_rng(batch)
+ seq_len = 53
+ feed = {
+ "input_ids": rng.integers(5, 1000, (batch, seq_len)),
+ "attention_mask": np.ones((batch, seq_len), dtype=np.int64),
+ "marker_pos": np.tile(np.array([[3, 9, 14, 20, 30]]), (batch, 1)),
+ "marker_mask": np.ones((batch, 5), dtype=bool),
+ "qtype": np.arange(batch, dtype=np.int64) % 3,
}
-
- # 4. Predict with both agents
- res_pt = agent_pt.predict(state, questions)
- res_onnx = agent_onnx.predict(state, questions)
-
- # 5. Assert parity for choice question
- assert res_pt["answers"]["intent"]["choice"] == res_onnx["answers"]["intent"]["choice"], \
- f"Choice mismatch: {res_pt['answers']['intent']['choice']} vs {res_onnx['answers']['intent']['choice']}"
-
- probs_pt = res_pt["answers"]["intent"]["probabilities"]
- probs_onnx = res_onnx["answers"]["intent"]["probabilities"]
- for k in probs_pt.keys():
- np.testing.assert_allclose(probs_pt[k], probs_onnx[k], atol=1e-3, rtol=1e-3)
-
- # 6. Assert parity for score question
- np.testing.assert_allclose(
- res_pt["answers"]["complexity"]["score"],
- res_onnx["answers"]["complexity"]["score"],
- atol=1e-3, rtol=1e-3,
- )
-
- probs_pt_s = res_pt["answers"]["complexity"]["probabilities"]
- probs_onnx_s = res_onnx["answers"]["complexity"]["probabilities"]
- for k in probs_pt_s.keys():
- np.testing.assert_allclose(probs_pt_s[k], probs_onnx_s[k], atol=1e-3, rtol=1e-3)
-
- # 7. Assert parity for noul question
- np.testing.assert_allclose(
- res_pt["answers"]["safety"]["noul"],
- res_onnx["answers"]["safety"]["noul"],
- atol=1e-3, rtol=1e-3,
- )
-
- # 8. The act head: its output was a static (1, 2) when the export traced at batch 1
- # (#695), and nothing above reads it.
- for qid in questions:
- np.testing.assert_allclose(
- res_pt["answers"][qid]["action"]["act_probability"],
- res_onnx["answers"][qid]["action"]["act_probability"],
- atol=5e-3,
- )
+ feed["attention_mask"][0, 40:] = 0
+ logits, act_logits = session.run(["logits", "act_logits"], feed)
+ assert logits.shape == (batch, 5) and act_logits.shape == (batch, 2), (logits.shape, act_logits.shape)
+ with torch.no_grad():
+ ref = agent_pt.model(*[torch.from_numpy(feed[n]) for n in INPUT_NAMES])
+ np.testing.assert_allclose(logits, ref[0].numpy(), atol=1e-3)
+ np.testing.assert_allclose(act_logits, ref[1].numpy(), atol=1e-2)
- # 9. The graph itself at batch 1 and 3 with a padded row, against the PyTorch model.
- # ONNXAgent collates one row per question, so the predict() above already ran at batch 3;
- # this pins the shapes and a padded key mask without any laya collation in the loop.
- for batch in (1, 3):
- rng = np.random.default_rng(batch)
- seq_len = 53
- feed = {
- "input_ids": rng.integers(5, 1000, (batch, seq_len)),
- "attention_mask": np.ones((batch, seq_len), dtype=np.int64),
- "marker_pos": np.tile(np.array([[3, 9, 14, 20, 30]]), (batch, 1)),
- "marker_mask": np.ones((batch, 5), dtype=bool),
- "qtype": np.arange(batch, dtype=np.int64) % 3,
+
[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")
+def test_onnx_numerical_parity(exported):
+ """Verify that PyTorch and ONNX agents produce identical outputs for all 3 question types."""
+ onnx_path, agent_pt = exported
+ agent_onnx = ONNXAgent(MODEL_ID, onnx_path)
+
+ # 3. Define a state and all 3 question types
+ state = {"request": "Refactor this service using dependency injection"}
+ questions = {
+ "intent": {
+ "type": "choice",
+ "instructions": "What is the user asking to do?",
+ "criteria": {
+ "refactor": "code refactoring, rewriting",
+ "bug_fix": "fixing bugs, issues",
+ "feature": "adding new features"
}
- feed["attention_mask"][0, 40:] = 0
- logits, act_logits = agent_onnx.session.run(["logits", "act_logits"], feed)
- assert logits.shape == (batch, 5) and act_logits.shape == (batch, 2), (logits.shape, act_logits.shape)
- with torch.no_grad():
- ref = agent_pt.model(*[torch.from_numpy(feed[n]) for n in
- ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")])
- np.testing.assert_allclose(logits, ref[0].numpy(), atol=1e-3)
- np.testing.assert_allclose(act_logits, ref[1].numpy(), atol=1e-2)
+ },
+ "complexity": {
+ "type": "score",
+ "instructions": "How complex is this request?",
+ "criteria": ["trivial", "simple", "moderate", "complex", "very complex"]
+ },
+ "safety": {
+ "type": "noul",
+ "instructions": "Does this request involve any unsafe or harmful content?"
+ },
+ }
+
+ # 4. Predict with both agents
+ res_pt = agent_pt.predict(state, questions)
+ res_onnx = agent_onnx.predict(state, questions)
+
+ # 5. Assert parity for choice question
+ assert res_pt["answers"]["intent"]["choice"] == res_onnx["answers"]["intent"]["choice"], \
+ f"Choice mismatch: {res_pt['answers']['intent']['choice']} vs {res_onnx['answers']['intent']['choice']}"
+
+ probs_pt = res_pt["answers"]["intent"]["probabilities"]
+ probs_onnx = res_onnx["answers"]["intent"]["probabilities"]
+ for k in probs_pt.keys():
+ np.testing.assert_allclose(probs_pt[k], probs_onnx[k], atol=1e-3, rtol=1e-3)
+
+ # 6. Assert parity for score question
+ np.testing.assert_allclose(
+ res_pt["answers"]["complexity"]["score"],
+ res_onnx["answers"]["complexity"]["score"],
+ atol=1e-3, rtol=1e-3,
+ )
+
+ probs_pt_s = res_pt["answers"]["complexity"]["probabilities"]
+ probs_onnx_s = res_onnx["answers"]["complexity"]["probabilities"]
+ for k in probs_pt_s.keys():
+ np.testing.assert_allclose(probs_pt_s[k], probs_onnx_s[k], atol=1e-3, rtol=1e-3)
+
+ # 7. Assert parity for noul question
+ np.testing.assert_allclose(
+ res_pt["answers"]["safety"]["noul"],
+ res_onnx["answers"]["safety"]["noul"],
+ atol=1e-3, rtol=1e-3,
+ )
+
+ # 8. The act head: its output was a static (1, 2) when the export traced at batch 1
+ # (#695), and nothing above reads it.
+ for qid in questions:
+ np.testing.assert_allclose(
+ res_pt["answers"][qid]["action"]["act_probability"],
+ res_onnx["answers"][qid]["action"]["act_probability"],
+ atol=5e-3,
+ )
- print("ONNX numerical parity test passed for all 3 question types!")
+ print("ONNX numerical parity test passed for all 3 question types!")