Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(onnx): export at batch 2 with explicit Dims so the graph runs at any batch size

cklxx0e88902b9c62
diff --git a/scripts/export_onnx.py b/scripts/export_onnx.py
index d71e3b02..7d0cf572 100644
--- a/scripts/export_onnx.py
+++ b/scripts/export_onnx.py
@@ -50,18 +50,17 @@ def export_to_onnx(model_id_or_path: str, output_path: str):
agent = Agent(model_id_or_path, compile=False, device="cpu")
print("Creating dummy input tensors...")
- # 1. Dummy tensors for tracing
- # (batch_size=1, seq_len=16)
- dummy_input_ids = torch.randint(0, 100, (1, 16), dtype=torch.long)
- dummy_attention_mask = torch.ones((1, 16), dtype=torch.long)
-
- # (batch_size=1, num_markers=2)
- dummy_marker_pos = torch.tensor([[1, 5]], dtype=torch.long)
- dummy_marker_mask = torch.tensor([[True, True]], dtype=torch.bool)
-
- # (batch_size=1)
- dummy_qtype = torch.tensor([0], dtype=torch.long)
-
+ # 1. Dummy tensors for tracing. torch.export specialises any dimension that is 1 (or equal
+ # to another) at trace time, so a batch-1 dummy baked batch=1 into the decision head's
+ # attention: the export ran at batch 1 and failed at batch >= 2 (#695). Batch, sequence and
+ # marker counts are therefore all > 1 and pairwise different. laya-ts's exporter does the same.
+ batch, seq_len, num_markers = 2, 17, 3
+ dummy_input_ids = torch.randint(0, 100, (batch, seq_len), dtype=torch.long)
+ dummy_attention_mask = torch.ones((batch, seq_len), dtype=torch.long)
+ dummy_marker_pos = torch.tensor([[1, 5, 9]] * batch, dtype=torch.long)
+ dummy_marker_mask = torch.ones((batch, num_markers), dtype=torch.bool)
+ dummy_qtype = torch.zeros(batch, dtype=torch.long)
+
inputs = (
dummy_input_ids,
dummy_attention_mask,
@@ -70,16 +69,19 @@ def export_to_onnx(model_id_or_path: str, output_path: str):
dummy_qtype,
)
- # 2. Define dynamic axes so the model can accept variable batch sizes and sequence lengths
- dynamic_axes = {
- "input_ids": {0: "batch_size", 1: "seq_len"},
- "attention_mask": {0: "batch_size", 1: "seq_len"},
- "marker_pos": {0: "batch_size", 1: "num_markers"},
- "marker_mask": {0: "batch_size", 1: "num_markers"},
- "qtype": {0: "batch_size"},
- "logits": {0: "batch_size", 1: "num_markers"},
- "act_logits": {0: "batch_size"},
- }
+ # 2. Dynamic dimensions, declared as torch.export Dims (what the dynamo exporter reads;
+ # `dynamic_axes` is only converted to these with a deprecation warning). The outputs follow
+ # from the inputs, so `act_logits` is (batch_size, 2) instead of a static (1, 2).
+ batch_dim = torch.export.Dim("batch_size")
+ seq_dim = torch.export.Dim("seq_len")
+ marker_dim = torch.export.Dim("num_markers")
+ dynamic_shapes = (
+ {0: batch_dim, 1: seq_dim},
+ {0: batch_dim, 1: seq_dim},
+ {0: batch_dim, 1: marker_dim},
+ {0: batch_dim, 1: marker_dim},
+ {0: batch_dim},
+ )
input_names = [
"input_ids",
@@ -110,7 +112,7 @@ def export_to_onnx(model_id_or_path: str, output_path: str):
do_constant_folding=True,
input_names=input_names,
output_names=output_names,
- dynamic_axes=dynamic_axes,
+ dynamic_shapes=dynamic_shapes,
)
print(f"Successfully exported ONNX model to: {output_path}")
diff --git a/tests/test_onnx.py b/tests/test_onnx.py
index 5f35b59f..40fc9162 100644
--- a/tests/test_onnx.py
+++ b/tests/test_onnx.py
@@ -2,6 +2,7 @@ import os
import sys
import tempfile
import numpy as np
+import torch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -88,4 +89,35 @@ def test_onnx_numerical_parity():
atol=1e-3, rtol=1e-3,
)
+ # 8. The act head: its output was a static (1, 2) when the export traced at batch 1
+ # (#695), and nothing above reads it.
+ for qid in questions:
+ np.testing.assert_allclose(
+ res_pt["answers"][qid]["action"]["act_probability"],
+ res_onnx["answers"][qid]["action"]["act_probability"],
+ atol=5e-3,
+ )
+
+ # 9. The graph itself at batch 1 and 3 with a padded row, against the PyTorch model.
+ # ONNXAgent collates one row per question, so the predict() above already ran at batch 3;
+ # this pins the shapes and a padded key mask without any laya collation in the loop.
+ for batch in (1, 3):
+ rng = np.random.default_rng(batch)
+ seq_len = 53
+ feed = {
+ "input_ids": rng.integers(5, 1000, (batch, seq_len)),
+ "attention_mask": np.ones((batch, seq_len), dtype=np.int64),
+ "marker_pos": np.tile(np.array([[3, 9, 14, 20, 30]]), (batch, 1)),
+ "marker_mask": np.ones((batch, 5), dtype=bool),
+ "qtype": np.arange(batch, dtype=np.int64) % 3,
+ }
+ feed["attention_mask"][0, 40:] = 0
+ logits, act_logits = agent_onnx.session.run(["logits", "act_logits"], feed)
+ assert logits.shape == (batch, 5) and act_logits.shape == (batch, 2), (logits.shape, act_logits.shape)
+ with torch.no_grad():
+ ref = agent_pt.model(*[torch.from_numpy(feed[n]) for n in
+ ("input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype")])
+ np.testing.assert_allclose(logits, ref[0].numpy(), atol=1e-3)
+ np.testing.assert_allclose(act_logits, ref[1].numpy(), atol=1e-2)
+
print("ONNX numerical parity test passed for all 3 question types!")