Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat: add torch.compile support and ONNX Runtime inference engine (#215)
Swayam Patil60965402db9b
diff --git a/laya/agent.py b/laya/agent.pyindex 7a1ad6fa..7538ef34 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -145,6 +145,7 @@ class Agent:token: Optional[str] = None,subfolder: Optional[str] = None,fast: bool = False,+ compile: bool = False,):"""Load a Laya checkpoint.@@ -248,11 +249,15 @@ class Agent:# ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder.# That is a loss for the batch sizes Laya runs (a handful of questions per call) and can- # hang on some platforms, so keep the eager path.+ # hang on some platforms, so keep the eager path unless explicitly requested.try:- self.model.encoder.config.reference_compile = False+ self.model.encoder.config.reference_compile = compileexcept Exception:pass++ # The TileLang fast path replaces the forward itself, so it takes precedence over compile.+ if compile and not fast:+ self.model = torch.compile(self.model)# Keep what the checkpoint shipped for inspection, but only ever apply clamped values:# some buckets are fitted to sharpen rather than soften (see clamp_temperature).diff --git a/laya/onnx_agent.py b/laya/onnx_agent.pynew file mode 100644index 00000000..1f15444d--- /dev/null+++ b/laya/onnx_agent.py@@ -0,0 +1,199 @@+import json+import os+import warnings+from typing import Any, Dict, Optional, Union++import numpy as np++from laya.common import (+ QTYPES,+ build_sequence,+ collate_items,+ confidence_from_probs,+ render_options,+ temp_bucket,+ TEMP_MIN,+ TEMP_MAX,+ clamp_temperature,+)+++class ONNXAgent:+ """System 1 decision model runtime via ONNX: fast CPU-optimized decisions."""++ def __init__(+ self,+ model_id_or_path: str,+ onnx_path: str = "laya.onnx",+ subfolder: Optional[str] = None,+ ):+ """Load a Laya agent backed by ONNX Runtime.++ Args:+ model_id_or_path: HuggingFace Hub ID or local path to the original PyTorch checkpoint+ (used to load the tokenizer and config).+ onnx_path: Path to the exported .onnx file.+ subfolder: Optional subfolder if downloading from a repo bundle.+ """+ import onnxruntime as ort+ from transformers import AutoTokenizer++ model_dir = model_id_or_path+ if not os.path.exists(model_dir):+ if model_id_or_path.startswith(("/", "./", "../")) or os.path.isabs(model_id_or_path):+ raise FileNotFoundError(+ f"Local model path not found: {model_id_or_path!r}."+ )+ from huggingface_hub import snapshot_download++ prefix = f"{subfolder}/" if subfolder else ""+ kw = {+ "allow_patterns": [prefix + name for name in (+ "rl_agent_config.json", "tokenizer/*", "encoder/*",+ )],+ }+ model_dir = snapshot_download(model_id_or_path, **kw)++ if subfolder:+ model_dir = os.path.join(model_dir, subfolder)+ if not os.path.isdir(model_dir):+ raise FileNotFoundError(+ f"Subfolder {subfolder!r} not found in {model_id_or_path!r}."+ )++ cfg_path = os.path.join(model_dir, "rl_agent_config.json")+ if not os.path.exists(cfg_path):+ raise FileNotFoundError(+ f"Incompatible model: {model_id_or_path!r} does not contain 'rl_agent_config.json'."+ )++ with open(cfg_path) as f:+ self.cfg = json.load(f)++ if not os.path.exists(onnx_path):+ raise FileNotFoundError(+ f"ONNX model not found at {onnx_path!r}. Please run export_onnx.py first."+ )++ # Load Tokenizer+ tok_dir = os.path.join(model_dir, "tokenizer")+ self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))++ # Initialize ONNX Runtime Session (auto-detect GPU if available)+ available = ort.get_available_providers()+ providers = (+ ["CUDAExecutionProvider", "CPUExecutionProvider"]+ if "CUDAExecutionProvider" in available+ else ["CPUExecutionProvider"]+ )+ self.session = ort.InferenceSession(onnx_path, providers=providers)++ self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])+ self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})+ self.temperature = [clamp_temperature(t) for t in self.temperature_raw]+ self.temperature_by_options = {k: clamp_temperature(v)+ for k, v in self.temperature_by_options_raw.items()}+ rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items()+ if clamp_temperature(v) != float(v)]+ rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw)+ if clamp_temperature(t) != float(t)]+ if rejected:+ warnings.warn(+ "laya ONNX: this checkpoint ships temperatures outside [%g, %g] which would distort "+ "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."+ % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),+ RuntimeWarning, stacklevel=2)++ @staticmethod+ def _to_internal(qdef: Dict) -> Dict:+ t = qdef["type"]+ crit = qdef.get("criteria")+ if t == "choice" and isinstance(crit, list):+ crit = {c: None for c in crit}+ ins = qdef["instructions"]+ if not isinstance(ins, str):+ ins = json.dumps(ins)+ return {"t": t, "ins": ins, "crit": crit}++ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:+ ids = list(questions.keys())+ items = []+ max_len = self.cfg.get("max_len", 512)+ head_max_len = self.cfg.get("head_max_len", 192)++ for qid in ids:+ q = self._to_internal(questions[qid])+ seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len)+ if len(markers) != len(render_options(q)):+ raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})++ b = collate_items([items], self.tok.pad_token_id)++ # Prepare ONNX inputs as numpy arrays+ ort_inputs = {+ "input_ids": b["input_ids"].numpy().astype(np.int64),+ "attention_mask": b["attention_mask"].numpy().astype(np.int64),+ "marker_pos": b["marker_pos"].numpy().astype(np.int64),+ "marker_mask": b["marker_mask"].numpy().astype(bool),+ "qtype": b["qtype"].numpy().astype(np.int64),+ }++ # Run ONNX inference+ ort_outs = self.session.run(["logits", "act_logits"], ort_inputs)+ logits = ort_outs[0]+ act_logits = ort_outs[1]++ # Compute softmax for actions manually in numpy+ act_exp = np.exp(act_logits - np.max(act_logits, axis=-1, keepdims=True))+ act = act_exp / np.sum(act_exp, axis=-1, keepdims=True)++ answers = {}+ n_tokens = int(b["attention_mask"].sum())++ for r, qid in enumerate(ids):+ q = self._to_internal(questions[qid])+ k = len(items[r]["markers"])+ qt = QTYPES[q["t"]]+ t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])+ z = logits[r, :k] / t_scale+ p = np.exp(z - z.max())+ p = p / p.sum()++ conf_score = round(confidence_from_probs(p, k), 4)+ ext = {"act_probability": round(float(act[r, 0]), 4)}++ if q["t"] == "choice":+ keys = list(q["crit"].keys())+ answers[qid] = {+ "type": "choice",+ "choice": keys[int(p.argmax())],+ "probabilities": {kk: round(float(v), 4) for kk, v in zip(keys, p)},+ "confidence": conf_score,+ "action": ext,+ }+ elif q["t"] == "score":+ exp_score = float((np.arange(k) * p).sum())+ answers[qid] = {+ "type": "score",+ "score": round(exp_score, 4),+ "legend": {str(i): c for i, c in enumerate(q["crit"])},+ "probabilities": {str(i): round(float(v), 4) for i, v in enumerate(p)},+ "confidence": conf_score,+ "action": ext,+ }+ else:+ answers[qid] = {+ "type": "noul",+ "noul": round(float(p[1]), 4),+ "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),+ "action": ext,+ }++ return {+ "model": "laya-rl-agent-onnx",+ "answers": answers,+ "usage": {"input_tokens": n_tokens, "output_tokens": 0},+ }++ predict = system_onediff --git a/pyproject.toml b/pyproject.tomlindex 092e8931..1eed70f9 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -48,6 +48,8 @@ fast = ["tilelang>=0.1.14"]# `pip install laya[mcp]` adds the MCP stdio server (laya.mcp / the laya-mcp-server CLI).mcp = ["mcp>=2.2.0"]+onnx = ["onnx", "onnxruntime"]+[project.scripts]laya = "laya.cli:main"laya-serve = "laya.serve:main"diff --git a/scripts/export_onnx.py b/scripts/export_onnx.pynew file mode 100644index 00000000..b48d233a--- /dev/null+++ b/scripts/export_onnx.py@@ -0,0 +1,83 @@+import argparse+import os+import torch++from laya.agent import Agent++def export_to_onnx(model_id_or_path: str, output_path: str):+ print(f"Loading PyTorch Agent from: {model_id_or_path}")+ agent = Agent(model_id_or_path, compile=False, device="cpu")++ print("Creating dummy input tensors...")+ # 1. Dummy tensors for tracing+ # (batch_size=1, seq_len=16)+ dummy_input_ids = torch.randint(0, 100, (1, 16), dtype=torch.long)+ dummy_attention_mask = torch.ones((1, 16), dtype=torch.long)++ # (batch_size=1, num_markers=2)+ dummy_marker_pos = torch.tensor([[1, 5]], dtype=torch.long)+ dummy_marker_mask = torch.tensor([[True, True]], dtype=torch.bool)++ # (batch_size=1)+ dummy_qtype = torch.tensor([0], dtype=torch.long)++ inputs = (+ dummy_input_ids,+ dummy_attention_mask,+ dummy_marker_pos,+ dummy_marker_mask,+ dummy_qtype,+ )++ # 2. Define dynamic axes so the model can accept variable batch sizes and sequence lengths+ dynamic_axes = {+ "input_ids": {0: "batch_size", 1: "seq_len"},+ "attention_mask": {0: "batch_size", 1: "seq_len"},+ "marker_pos": {0: "batch_size", 1: "num_markers"},+ "marker_mask": {0: "batch_size", 1: "num_markers"},+ "qtype": {0: "batch_size"},+ "logits": {0: "batch_size", 1: "num_markers"},+ "act_logits": {0: "batch_size"},+ }++ input_names = [+ "input_ids",+ "attention_mask",+ "marker_pos",+ "marker_mask",+ "qtype",+ ]++ output_names = ["logits", "act_logits"]++ print(f"Exporting to {output_path} (this may take a minute)...")++ out_dir = os.path.dirname(os.path.abspath(output_path))+ if out_dir:+ os.makedirs(out_dir, exist_ok=True)++ # We must detach the encoder because ONNX export runs the model in trace mode.+ # The `detach_encoder` flag in forward() just detaches the hidden state gradient,+ # but we don't even need to pass it since kwargs are ignored by tracing.++ torch.onnx.export(+ agent.model,+ inputs,+ output_path,+ export_params=True,+ opset_version=18,+ do_constant_folding=True,+ input_names=input_names,+ output_names=output_names,+ dynamic_axes=dynamic_axes,+ )++ print(f"Successfully exported ONNX model to: {output_path}")++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description="Export a Laya model to ONNX format")+ parser.add_argument("--model", type=str, default="convaiinnovations/laya", help="HuggingFace Hub ID or local path")+ parser.add_argument("--output", type=str, default="laya.onnx", help="Output path for the ONNX file")+ args = parser.parse_args()++ export_to_onnx(args.model, args.output)diff --git a/tests/test_onnx.py b/tests/test_onnx.pynew file mode 100644index 00000000..5f35b59f--- /dev/null+++ b/tests/test_onnx.py@@ -0,0 +1,91 @@+import os+import sys+import tempfile+import numpy as np++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.agent import Agent++# Skip the test if onnx isn't installed+try:+ import onnxruntime+ from laya.onnx_agent import ONNXAgent+ from scripts.export_onnx import export_to_onnx+ HAS_ONNX = True+except ImportError:+ HAS_ONNX = False++import pytest+[email protected](not HAS_ONNX, reason="onnx and onnxruntime are required")+def test_onnx_numerical_parity():+ """Verify that PyTorch and ONNX agents produce identical outputs for all 3 question types."""+ model_id = "convaiinnovations/laya"++ with tempfile.TemporaryDirectory() as tmpdir:+ onnx_path = os.path.join(tmpdir, "laya.onnx")++ # 1. Export to ONNX+ export_to_onnx(model_id, onnx_path)++ # 2. Load both PyTorch and ONNX Agents+ agent_pt = Agent(model_id, compile=False, device="cpu")+ agent_onnx = ONNXAgent(model_id, onnx_path)++ # 3. Define a state and all 3 question types+ state = {"request": "Refactor this service using dependency injection"}+ questions = {+ "intent": {+ "type": "choice",+ "instructions": "What is the user asking to do?",+ "criteria": {+ "refactor": "code refactoring, rewriting",+ "bug_fix": "fixing bugs, issues",+ "feature": "adding new features"+ }+ },+ "complexity": {+ "type": "score",+ "instructions": "How complex is this request?",+ "criteria": ["trivial", "simple", "moderate", "complex", "very complex"]+ },+ "safety": {+ "type": "noul",+ "instructions": "Does this request involve any unsafe or harmful content?"+ },+ }++ # 4. Predict with both agents+ res_pt = agent_pt.predict(state, questions)+ res_onnx = agent_onnx.predict(state, questions)++ # 5. Assert parity for choice question+ assert res_pt["answers"]["intent"]["choice"] == res_onnx["answers"]["intent"]["choice"], \+ f"Choice mismatch: {res_pt['answers']['intent']['choice']} vs {res_onnx['answers']['intent']['choice']}"++ probs_pt = res_pt["answers"]["intent"]["probabilities"]+ probs_onnx = res_onnx["answers"]["intent"]["probabilities"]+ for k in probs_pt.keys():+ np.testing.assert_allclose(probs_pt[k], probs_onnx[k], atol=1e-3, rtol=1e-3)++ # 6. Assert parity for score question+ np.testing.assert_allclose(+ res_pt["answers"]["complexity"]["score"],+ res_onnx["answers"]["complexity"]["score"],+ atol=1e-3, rtol=1e-3,+ )++ probs_pt_s = res_pt["answers"]["complexity"]["probabilities"]+ probs_onnx_s = res_onnx["answers"]["complexity"]["probabilities"]+ for k in probs_pt_s.keys():+ np.testing.assert_allclose(probs_pt_s[k], probs_onnx_s[k], atol=1e-3, rtol=1e-3)++ # 7. Assert parity for noul question+ np.testing.assert_allclose(+ res_pt["answers"]["safety"]["noul"],+ res_onnx["answers"]["safety"]["noul"],+ atol=1e-3, rtol=1e-3,+ )++ print("ONNX numerical parity test passed for all 3 question types!")