Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat: add torch.compile support and ONNX Runtime inference engine (#215)

Swayam Patil60965402db9b
diff --git a/laya/agent.py b/laya/agent.py
index 7a1ad6fa..7538ef34 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -145,6 +145,7 @@ class Agent:
token: Optional[str] = None,
subfolder: Optional[str] = None,
fast: bool = False,
+ compile: bool = False,
):
"""Load a Laya checkpoint.
@@ -248,11 +249,15 @@ class Agent:
# ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder.
# That is a loss for the batch sizes Laya runs (a handful of questions per call) and can
- # hang on some platforms, so keep the eager path.
+ # hang on some platforms, so keep the eager path unless explicitly requested.
try:
- self.model.encoder.config.reference_compile = False
+ self.model.encoder.config.reference_compile = compile
except Exception:
pass
+
+ # The TileLang fast path replaces the forward itself, so it takes precedence over compile.
+ if compile and not fast:
+ self.model = torch.compile(self.model)
# Keep what the checkpoint shipped for inspection, but only ever apply clamped values:
# some buckets are fitted to sharpen rather than soften (see clamp_temperature).
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
new file mode 100644
index 00000000..1f15444d
--- /dev/null
+++ b/laya/onnx_agent.py
@@ -0,0 +1,199 @@
+import json
+import os
+import warnings
+from typing import Any, Dict, Optional, Union
+
+import numpy as np
+
+from laya.common import (
+ QTYPES,
+ build_sequence,
+ collate_items,
+ confidence_from_probs,
+ render_options,
+ temp_bucket,
+ TEMP_MIN,
+ TEMP_MAX,
+ clamp_temperature,
+)
+
+
+class ONNXAgent:
+ """System 1 decision model runtime via ONNX: fast CPU-optimized decisions."""
+
+ def __init__(
+ self,
+ model_id_or_path: str,
+ onnx_path: str = "laya.onnx",
+ subfolder: Optional[str] = None,
+ ):
+ """Load a Laya agent backed by ONNX Runtime.
+
+ Args:
+ model_id_or_path: HuggingFace Hub ID or local path to the original PyTorch checkpoint
+ (used to load the tokenizer and config).
+ onnx_path: Path to the exported .onnx file.
+ subfolder: Optional subfolder if downloading from a repo bundle.
+ """
+ import onnxruntime as ort
+ from transformers import AutoTokenizer
+
+ model_dir = model_id_or_path
+ if not os.path.exists(model_dir):
+ if model_id_or_path.startswith(("/", "./", "../")) or os.path.isabs(model_id_or_path):
+ raise FileNotFoundError(
+ f"Local model path not found: {model_id_or_path!r}."
+ )
+ from huggingface_hub import snapshot_download
+
+ prefix = f"{subfolder}/" if subfolder else ""
+ kw = {
+ "allow_patterns": [prefix + name for name in (
+ "rl_agent_config.json", "tokenizer/*", "encoder/*",
+ )],
+ }
+ model_dir = snapshot_download(model_id_or_path, **kw)
+
+ if subfolder:
+ model_dir = os.path.join(model_dir, subfolder)
+ if not os.path.isdir(model_dir):
+ raise FileNotFoundError(
+ f"Subfolder {subfolder!r} not found in {model_id_or_path!r}."
+ )
+
+ cfg_path = os.path.join(model_dir, "rl_agent_config.json")
+ if not os.path.exists(cfg_path):
+ raise FileNotFoundError(
+ f"Incompatible model: {model_id_or_path!r} does not contain 'rl_agent_config.json'."
+ )
+
+ with open(cfg_path) as f:
+ self.cfg = json.load(f)
+
+ if not os.path.exists(onnx_path):
+ raise FileNotFoundError(
+ f"ONNX model not found at {onnx_path!r}. Please run export_onnx.py first."
+ )
+
+ # Load Tokenizer
+ tok_dir = os.path.join(model_dir, "tokenizer")
+ self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))
+
+ # Initialize ONNX Runtime Session (auto-detect GPU if available)
+ available = ort.get_available_providers()
+ providers = (
+ ["CUDAExecutionProvider", "CPUExecutionProvider"]
+ if "CUDAExecutionProvider" in available
+ else ["CPUExecutionProvider"]
+ )
+ self.session = ort.InferenceSession(onnx_path, providers=providers)
+
+ self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])
+ self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})
+ self.temperature = [clamp_temperature(t) for t in self.temperature_raw]
+ self.temperature_by_options = {k: clamp_temperature(v)
+ for k, v in self.temperature_by_options_raw.items()}
+ rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items()
+ if clamp_temperature(v) != float(v)]
+ rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw)
+ if clamp_temperature(t) != float(t)]
+ if rejected:
+ warnings.warn(
+ "laya ONNX: this checkpoint ships temperatures outside [%g, %g] which would distort "
+ "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."
+ % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
+ RuntimeWarning, stacklevel=2)
+
+ @staticmethod
+ def _to_internal(qdef: Dict) -> Dict:
+ t = qdef["type"]
+ crit = qdef.get("criteria")
+ if t == "choice" and isinstance(crit, list):
+ crit = {c: None for c in crit}
+ ins = qdef["instructions"]
+ if not isinstance(ins, str):
+ ins = json.dumps(ins)
+ return {"t": t, "ins": ins, "crit": crit}
+
+ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
+ ids = list(questions.keys())
+ items = []
+ max_len = self.cfg.get("max_len", 512)
+ head_max_len = self.cfg.get("head_max_len", 192)
+
+ for qid in ids:
+ q = self._to_internal(questions[qid])
+ seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len)
+ if len(markers) != len(render_options(q)):
+ raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})
+
+ b = collate_items([items], self.tok.pad_token_id)
+
+ # Prepare ONNX inputs as numpy arrays
+ ort_inputs = {
+ "input_ids": b["input_ids"].numpy().astype(np.int64),
+ "attention_mask": b["attention_mask"].numpy().astype(np.int64),
+ "marker_pos": b["marker_pos"].numpy().astype(np.int64),
+ "marker_mask": b["marker_mask"].numpy().astype(bool),
+ "qtype": b["qtype"].numpy().astype(np.int64),
+ }
+
+ # Run ONNX inference
+ ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)
+ logits = ort_outs[0]
+ act_logits = ort_outs[1]
+
+ # Compute softmax for actions manually in numpy
+ act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))
+ act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)
+
+ answers = {}
+ n_tokens = int(b["attention_mask"].sum())
+
+ for r, qid in enumerate(ids):
+ q = self._to_internal(questions[qid])
+ k = len(items[r]["markers"])
+ qt = QTYPES[q["t"]]
+ t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])
+ z = logits[r, :k] / t_scale
+ p = np.exp(z - z.max())
+ p = p / p.sum()
+
+ conf_score = round(confidence_from_probs(p, k), 4)
+ ext = {"act_probability": round(float(act[r, 0]), 4)}
+
+ if q["t"] == "choice":
+ keys = list(q["crit"].keys())
+ answers[qid] = {
+ "type": "choice",
+ "choice": keys[int(p.argmax())],
+ "probabilities": {kk: round(float(v), 4) for kk, v in zip(keys, p)},
+ "confidence": conf_score,
+ "action": ext,
+ }
+ elif q["t"] == "score":
+ exp_score = float((np.arange(k) * p).sum())
+ answers[qid] = {
+ "type": "score",
+ "score": round(exp_score, 4),
+ "legend": {str(i): c for i, c in enumerate(q["crit"])},
+ "probabilities": {str(i): round(float(v), 4) for i, v in enumerate(p)},
+ "confidence": conf_score,
+ "action": ext,
+ }
+ else:
+ answers[qid] = {
+ "type": "noul",
+ "noul": round(float(p[1]), 4),
+ "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),
+ "action": ext,
+ }
+
+ return {
+ "model": "laya-rl-agent-onnx",
+ "answers": answers,
+ "usage": {"input_tokens": n_tokens, "output_tokens": 0},
+ }
+
+ predict = system_one
diff --git a/pyproject.toml b/pyproject.toml
index 092e8931..1eed70f9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,8 @@ fast = ["tilelang>=0.1.14"]
# `pip install laya[mcp]` adds the MCP stdio server (laya.mcp / the laya-mcp-server CLI).
mcp = ["mcp>=2.2.0"]
+onnx = ["onnx", "onnxruntime"]
+
[project.scripts]
laya = "laya.cli:main"
laya-serve = "laya.serve:main"
diff --git a/scripts/export_onnx.py b/scripts/export_onnx.py
new file mode 100644
index 00000000..b48d233a
--- /dev/null
+++ b/scripts/export_onnx.py
@@ -0,0 +1,83 @@
+import argparse
+import os
+import torch
+
+from laya.agent import Agent
+
+def export_to_onnx(model_id_or_path: str, output_path: str):
+ print(f"Loading PyTorch Agent from: {model_id_or_path}")
+ agent = Agent(model_id_or_path, compile=False, device="cpu")
+
+ print("Creating dummy input tensors...")
+ # 1. Dummy tensors for tracing
+ # (batch_size=1, seq_len=16)
+ dummy_input_ids = torch.randint(0, 100, (1, 16), dtype=torch.long)
+ dummy_attention_mask = torch.ones((1, 16), dtype=torch.long)
+
+ # (batch_size=1, num_markers=2)
+ dummy_marker_pos = torch.tensor([[1, 5]], dtype=torch.long)
+ dummy_marker_mask = torch.tensor([[True, True]], dtype=torch.bool)
+
+ # (batch_size=1)
+ dummy_qtype = torch.tensor([0], dtype=torch.long)
+
+ inputs = (
+ dummy_input_ids,
+ dummy_attention_mask,
+ dummy_marker_pos,
+ dummy_marker_mask,
+ dummy_qtype,
+ )
+
+ # 2. Define dynamic axes so the model can accept variable batch sizes and sequence lengths
+ dynamic_axes = {
+ "input_ids": {0: "batch_size", 1: "seq_len"},
+ "attention_mask": {0: "batch_size", 1: "seq_len"},
+ "marker_pos": {0: "batch_size", 1: "num_markers"},
+ "marker_mask": {0: "batch_size", 1: "num_markers"},
+ "qtype": {0: "batch_size"},
+ "logits": {0: "batch_size", 1: "num_markers"},
+ "act_logits": {0: "batch_size"},
+ }
+
+ input_names = [
+ "input_ids",
+ "attention_mask",
+ "marker_pos",
+ "marker_mask",
+ "qtype",
+ ]
+
+ output_names = ["logits", "act_logits"]
+
+ print(f"Exporting to {output_path} (this may take a minute)...")
+
+ out_dir = os.path.dirname(os.path.abspath(output_path))
+ if out_dir:
+ os.makedirs(out_dir, exist_ok=True)
+
+ # We must detach the encoder because ONNX export runs the model in trace mode.
+ # The `detach_encoder` flag in forward() just detaches the hidden state gradient,
+ # but we don't even need to pass it since kwargs are ignored by tracing.
+
+ torch.onnx.export(
+ agent.model,
+ inputs,
+ output_path,
+ export_params=True,
+ opset_version=18,
+ do_constant_folding=True,
+ input_names=input_names,
+ output_names=output_names,
+ dynamic_axes=dynamic_axes,
+ )
+
+ print(f"Successfully exported ONNX model to: {output_path}")
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Export a Laya model to ONNX format")
+ parser.add_argument("--model", type=str, default="convaiinnovations/laya", help="HuggingFace Hub ID or local path")
+ parser.add_argument("--output", type=str, default="laya.onnx", help="Output path for the ONNX file")
+ args = parser.parse_args()
+
+ export_to_onnx(args.model, args.output)
diff --git a/tests/test_onnx.py b/tests/test_onnx.py
new file mode 100644
index 00000000..5f35b59f
--- /dev/null
+++ b/tests/test_onnx.py
@@ -0,0 +1,91 @@
+import os
+import sys
+import tempfile
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.agent import Agent
+
+# Skip the test if onnx isn't installed
+try:
+ import onnxruntime
+ from laya.onnx_agent import ONNXAgent
+ from scripts.export_onnx import export_to_onnx
+ HAS_ONNX = True
+except ImportError:
+ HAS_ONNX = False
+
+import pytest
+
[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")
+def test_onnx_numerical_parity():
+ """Verify that PyTorch and ONNX agents produce identical outputs for all 3 question types."""
+ model_id = "convaiinnovations/laya"
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ onnx_path = os.path.join(tmpdir, "laya.onnx")
+
+ # 1. Export to ONNX
+ export_to_onnx(model_id, onnx_path)
+
+ # 2. Load both PyTorch and ONNX Agents
+ agent_pt = Agent(model_id, compile=False, device="cpu")
+ agent_onnx = ONNXAgent(model_id, onnx_path)
+
+ # 3. Define a state and all 3 question types
+ state = {"request": "Refactor this service using dependency injection"}
+ questions = {
+ "intent": {
+ "type": "choice",
+ "instructions": "What is the user asking to do?",
+ "criteria": {
+ "refactor": "code refactoring, rewriting",
+ "bug_fix": "fixing bugs, issues",
+ "feature": "adding new features"
+ }
+ },
+ "complexity": {
+ "type": "score",
+ "instructions": "How complex is this request?",
+ "criteria": ["trivial", "simple", "moderate", "complex", "very complex"]
+ },
+ "safety": {
+ "type": "noul",
+ "instructions": "Does this request involve any unsafe or harmful content?"
+ },
+ }
+
+ # 4. Predict with both agents
+ res_pt = agent_pt.predict(state, questions)
+ res_onnx = agent_onnx.predict(state, questions)
+
+ # 5. Assert parity for choice question
+ assert res_pt["answers"]["intent"]["choice"] == res_onnx["answers"]["intent"]["choice"], \
+ f"Choice mismatch: {res_pt['answers']['intent']['choice']} vs {res_onnx['answers']['intent']['choice']}"
+
+ probs_pt = res_pt["answers"]["intent"]["probabilities"]
+ probs_onnx = res_onnx["answers"]["intent"]["probabilities"]
+ for k in probs_pt.keys():
+ np.testing.assert_allclose(probs_pt[k], probs_onnx[k], atol=1e-3, rtol=1e-3)
+
+ # 6. Assert parity for score question
+ np.testing.assert_allclose(
+ res_pt["answers"]["complexity"]["score"],
+ res_onnx["answers"]["complexity"]["score"],
+ atol=1e-3, rtol=1e-3,
+ )
+
+ probs_pt_s = res_pt["answers"]["complexity"]["probabilities"]
+ probs_onnx_s = res_onnx["answers"]["complexity"]["probabilities"]
+ for k in probs_pt_s.keys():
+ np.testing.assert_allclose(probs_pt_s[k], probs_onnx_s[k], atol=1e-3, rtol=1e-3)
+
+ # 7. Assert parity for noul question
+ np.testing.assert_allclose(
+ res_pt["answers"]["safety"]["noul"],
+ res_onnx["answers"]["safety"]["noul"],
+ atol=1e-3, rtol=1e-3,
+ )
+
+ print("ONNX numerical parity test passed for all 3 question types!")