Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): verify the ONNX graphs before anything lands in --out-dir
Aashish58f4b16e8f10
diff --git a/laya-ts/scripts/export_onnx.py b/laya-ts/scripts/export_onnx.pynew file mode 100644index 00000000..85c57efc--- /dev/null+++ b/laya-ts/scripts/export_onnx.py@@ -0,0 +1,263 @@+"""Export a Laya checkpoint to split ONNX (encoder.onnx + head.onnx).++Source of truth stays ``model.safetensors``; this script produces the two ONNX+files the ``laya-ts`` Node/Web providers load. Run once per checkpoint:+ python laya-ts/scripts/export_onnx.py --repo convaiinnovations/laya --out-dir ./model+ python laya-ts/scripts/export_onnx.py --repo convaiinnovations/laya --subfolder multilingual --out-dir ./model-ml+ python laya-ts/scripts/export_onnx.py --model-dir <local ckpt> --out-dir ./model++With ``--repo`` the checkpoint is downloaded from Hugging Face (safetensors ++tokenizer + encoder) into the HF cache first, then converted. With+``--model-dir`` an already-downloaded directory is used directly.++The encoder graph takes ``(input_ids, attention_mask)`` and returns+``last_hidden_state``. The head graph takes ``(hidden_states, marker_pos,+marker_mask, qtype)`` and returns ``(logits, act_logits)``. Verification runs+a torch forward at ``--seq-len`` and at ``--verify-len`` (default 512, past the+local_attention=128 sliding window) and requires the ONNX outputs to match+within 1e-4 at both lengths before anything is written.+"""+import argparse+import json+import os+import shutil+import tempfile++try:+ from laya.common import build_model+except ImportError: # --help must work without torch installed+ build_model = None++DEFAULT_REPO = "convaiinnovations/laya"+++def _download(repo, subfolder=None, token=None):+ """Download a checkpoint from HF into the HF cache; return the checkpoint dir."""+ # Windows without Developer Mode can't symlink the HF cache; the hub prints+ # a long warning and falls back to copies. Silence it, copies still work.+ os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")+ try:+ from huggingface_hub import snapshot_download+ except ImportError:+ raise SystemExit("download needs huggingface_hub: pip install huggingface_hub")+ prefix = ("%s/" % subfolder) if subfolder else ""+ allow = [prefix + name for name in (+ "rl_agent_config.json", "model.safetensors", "config.json",+ "tokenizer.json", "tokenizer/*", "encoder/*",+ )]+ root = snapshot_download(repo, allow_patterns=allow,+ token=token or os.environ.get("HF_TOKEN"))+ path = os.path.join(root, subfolder) if subfolder else root+ if not os.path.isdir(path):+ raise SystemExit("subfolder %r not found in %r" % (subfolder, repo))+ return path+++def _ensure_tokenizer_json(model_dir):+ """Copies tokenizer/tokenizer.json up to the checkpoint root when needed."""+ dest = os.path.join(model_dir, "tokenizer.json")+ if not os.path.exists(dest):+ src = os.path.join(model_dir, "tokenizer", "tokenizer.json")+ if os.path.exists(src):+ shutil.copy(src, dest)+++def _checkpoint_weights_path(model_dir):+ """Require the checkpoint's safetensors weights before building a model."""+ path = os.path.join(model_dir, "model.safetensors")+ if not os.path.isfile(path):+ raise SystemExit("checkpoint is missing model.safetensors: %s" % model_dir)+ return path+++def parse_args(argv=None):+ p = argparse.ArgumentParser(description=__doc__)+ src = p.add_mutually_exclusive_group(required=True)+ src.add_argument("--repo", help="HF repo to download first (default %s with --model-dir absent)." % DEFAULT_REPO)+ src.add_argument("--model-dir", help="Local checkpoint dir with rl_agent_config.json (+ tokenizer.json).")+ p.add_argument("--subfolder", default=None, help="Subfolder inside the repo (e.g. multilingual, typed-decisions).")+ p.add_argument("--token", default=None, help="HF token (or HF_TOKEN env).")+ p.add_argument("--out-dir", required=True, help="Output dir for encoder.onnx, head.onnx, tokenizer.json, rl_agent_config.json.")+ p.add_argument("--opset", type=int, default=18, help="ONNX opset (default 18; the exporter implements 18 natively and a downgrade to 17 emits invalid Split nodes).")+ p.add_argument("--seq-len", type=int, default=16, help="Dummy sequence length for tracing/verification (default 16).")+ p.add_argument("--verify-len", type=int, default=512, help="Second length for the torch-vs-ONNX 1e-4 check (default 512; exercises sliding-window layers, local_attention=128).")+ p.add_argument("--no-verify", action="store_true", help="Skip the torch-vs-ONNX 1e-4 check.")+ return p.parse_args(argv)+++def main(argv=None):+ args = parse_args(argv)+ if build_model is None:+ raise SystemExit("export needs torch + laya installed: pip install torch transformers onnx onnxruntime")+ model_dir = args.model_dir or _download(args.repo or DEFAULT_REPO, args.subfolder, args.token)+ weights_path = _checkpoint_weights_path(model_dir)+ _ensure_tokenizer_json(model_dir)+ import torch++ class _EncoderOnly(torch.nn.Module):+ def __init__(self, model):+ super().__init__()+ self.encoder = model.encoder++ def forward(self, input_ids, attention_mask):+ return self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state++ class _HeadOnly(torch.nn.Module):+ """Decision head tail: type_emb + transformer head + scorer + act_head."""++ def __init__(self, model):+ super().__init__()+ self.head = model.head+ self.type_emb = model.type_emb+ self.scorer = model.scorer+ self.act_head = model.act_head++ def forward(self, hidden_states, marker_pos, marker_mask, qtype, attention_mask):+ h = hidden_states + self.type_emb(qtype.squeeze(-1))[:, None, :]+ if self.head is not None:+ # Same as DecisionModel.forward: padding must not be attended.+ # Without this, batch mates of unequal length corrupt each+ # other's markers through the head transformer.+ pad = attention_mask == 0+ for layer in self.head.layers:+ h = layer(h, src_key_padding_mask=pad)+ idx = marker_pos.clamp(min=0)[:, :, None].expand(-1, -1, h.size(-1))+ m = torch.gather(h, 1, idx)+ logits = self.scorer(m).squeeze(-1).float()+ logits = logits.masked_fill(~marker_mask, -1e4)+ p = torch.softmax(logits.detach(), -1)+ k = marker_mask.sum(-1).clamp(min=2).float()+ ent = -(p * torch.log(p.clamp_min(1e-9))).sum(-1) / torch.log(k)+ top2 = p.topk(2, -1).values+ feats = torch.stack([top2[:, 0], top2[:, 0] - top2[:, 1], ent, k / 255.0], -1)+ act_logits = self.act_head(torch.cat([h[:, 0].float(), feats], -1))+ return logits, act_logits++ with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f:+ cfg = json.load(f)+ # The encoder arch lives in <ckpt>/encoder/config.json (same as Agent:+ # enc_dir = model_dir/"encoder"); the checkpoint root itself has no+ # config.json, so passing model_dir here makes AutoConfig fail with+ # "Unrecognized model ... no model_type key".+ enc_dir = os.path.join(model_dir, "encoder")+ model = build_model(cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None,+ pretrained=False)+ from safetensors.torch import load_file as _load_safetensors++ model.load_state_dict(_load_safetensors(weights_path), strict=False)+ model.eval().float()++ # Batch must stay symbolic: tracing with batch=1 lets dynamo bake batch=1+ # into a head reshape (batch=2 then fails with "204 by 408" broadcast).+ # Explicit Dims + batch=2 dummies keep it dynamic; verify covers 1 and 2.+ batch_dim = torch.export.Dim("batch", min=1, max=64)+ seq_dim = torch.export.Dim("seq", min=1, max=8192)+ markers_dim = torch.export.Dim("markers", min=1, max=256)+ enc_shapes = ({0: batch_dim, 1: seq_dim}, {0: batch_dim, 1: seq_dim})+ head_shapes = ({0: batch_dim, 1: seq_dim}, {0: batch_dim, 1: markers_dim},+ {0: batch_dim, 1: markers_dim}, {0: batch_dim},+ {0: batch_dim, 1: seq_dim})++ seq = int(args.seq_len)+ input_ids = torch.ones(2, seq, dtype=torch.long)+ attention_mask = torch.ones(2, seq, dtype=torch.long)+ ref_hidden, ref_logits, ref_act, marker_pos, marker_mask, qtype, ref_att = _run_ref(model, _HeadOnly(model).eval(), seq, batch=2)++ # The module docstring promises the outputs match "within 1e-4 at both lengths before anything is+ # written", and verification reads the graphs back off disk, so "written" has to mean "written to+ # --out-dir". Tracing straight into it left two unverified graphs there on a failed run -- which is+ # exactly what the Node/Web providers then load -- and created --out-dir even when the run refused.+ staged = tempfile.mkdtemp(prefix="laya-onnx-")+ try:+ enc_path = os.path.join(staged, "encoder.onnx")+ head_path = os.path.join(staged, "head.onnx")++ torch.onnx.export(+ _EncoderOnly(model).eval(),+ (input_ids, attention_mask),+ enc_path,+ input_names=["input_ids", "attention_mask"],+ output_names=["last_hidden_state"],+ dynamic_axes={"input_ids": {0: "batch", 1: "seq"}, "attention_mask": {0: "batch", 1: "seq"}, "last_hidden_state": {0: "batch", 1: "seq"}},+ dynamic_shapes=enc_shapes,+ opset_version=args.opset,+ )+ torch.onnx.export(+ _HeadOnly(model).eval(),+ (ref_hidden, marker_pos, marker_mask, qtype, ref_att),+ head_path,+ input_names=["hidden_states", "marker_pos", "marker_mask", "qtype", "attention_mask"],+ output_names=["logits", "act_logits"],+ dynamic_axes={"hidden_states": {0: "batch", 1: "seq"}, "marker_pos": {0: "batch", 1: "markers"}, "marker_mask": {0: "batch", 1: "markers"}, "qtype": {0: "batch"}},+ dynamic_shapes=head_shapes,+ opset_version=args.opset,+ )++ if not args.no_verify:+ _verify(enc_path, head_path, input_ids, attention_mask, marker_pos, marker_mask, qtype, ref_att, ref_logits, ref_act)+ b1 = _run_ref(model, _HeadOnly(model).eval(), seq, batch=1)+ _verify(enc_path, head_path, torch.ones(1, seq, dtype=torch.long),+ torch.ones(1, seq, dtype=torch.long), *b1[3:6], b1[6], b1[1], b1[2])+ if int(args.verify_len) != seq:+ v_hidden, v_logits, v_act, v_pos, v_mask, v_qtype, v_att2 = _run_ref(model, _HeadOnly(model).eval(), int(args.verify_len), batch=2)+ v_ids = torch.ones(2, int(args.verify_len), dtype=torch.long)+ v_att = torch.ones(2, int(args.verify_len), dtype=torch.long)+ _verify(enc_path, head_path, v_ids, v_att, v_pos, v_mask, v_qtype, v_att2, v_logits, v_act)++ os.makedirs(args.out_dir, exist_ok=True)+ for name in ("encoder.onnx", "head.onnx"):+ shutil.move(os.path.join(staged, name), os.path.join(args.out_dir, name))+ for name in ("tokenizer.json", "rl_agent_config.json"):+ src = os.path.join(model_dir, name)+ if os.path.exists(src):+ shutil.copy(src, os.path.join(args.out_dir, name))+ print("wrote %s and %s" % (os.path.join(args.out_dir, "encoder.onnx"),+ os.path.join(args.out_dir, "head.onnx")))+ finally:+ shutil.rmtree(staged, ignore_errors=True)+++def _run_ref(model, head, seq, batch=1):+ import torch++ input_ids = torch.ones(batch, seq, dtype=torch.long)+ attention_mask = torch.ones(batch, seq, dtype=torch.long)+ with torch.inference_mode():+ ref_hidden = model.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state+ marker_pos = torch.tensor([[1, 2]] * batch, dtype=torch.long)+ marker_mask = torch.tensor([[True, True]] * batch)+ qtype = torch.tensor([[0]] * batch, dtype=torch.long)+ ref_logits, ref_act = head(ref_hidden, marker_pos, marker_mask, qtype, attention_mask)+ return ref_hidden, ref_logits, ref_act, marker_pos, marker_mask, qtype, attention_mask+++def _verify(enc_path, head_path, input_ids, attention_mask, marker_pos, marker_mask, qtype, head_att, ref_logits, ref_act):+ import numpy as np++ try:+ import onnxruntime as ort+ except ImportError:+ raise SystemExit("export --no-verify not given but onnxruntime is missing: pip install onnxruntime")+ enc = ort.InferenceSession(enc_path, providers=["CPUExecutionProvider"])+ head = ort.InferenceSession(head_path, providers=["CPUExecutionProvider"])+ (hidden,) = enc.run(None, {"input_ids": input_ids.numpy(), "attention_mask": attention_mask.numpy()})+ logits, act = head.run(None, {"hidden_states": hidden.astype(np.float32), "marker_pos": marker_pos.numpy(), "marker_mask": marker_mask.numpy(), "qtype": qtype.numpy(), "attention_mask": head_att.numpy()})+ diff = float(np.abs(logits - ref_logits.detach().numpy()).max())+ if diff > 1e-4:+ raise SystemExit("verification failed: logits max abs diff %.2e > 1e-4" % diff)+ # act_logits are consumed only through softmax (rounded to 4dp downstream),+ # and dynamo decompositions add ~5e-4 raw-logit noise that softmax washes+ # out. Compare the probabilities TS actually reads instead of raw logits.+ got_p = np.exp(act - act.max(axis=-1, keepdims=True))+ got_p /= got_p.sum(axis=-1, keepdims=True)+ ref = ref_act.detach().numpy()+ ref_p = np.exp(ref - ref.max(axis=-1, keepdims=True))+ ref_p /= ref_p.sum(axis=-1, keepdims=True)+ pdiff = float(np.abs(got_p - ref_p).max())+ if pdiff > 1e-4:+ raise SystemExit("verification failed: act_probs max abs diff %.2e > 1e-4" % pdiff)+ print("verify ok: torch vs onnx match within 1e-4 (logits %.1e, act_probs %.1e)" % (diff, pdiff))+++if __name__ == "__main__":+ main()diff --git a/tests/test_export_onnx_safety.py b/tests/test_export_onnx_safety.pynew file mode 100644index 00000000..5154d636--- /dev/null+++ b/tests/test_export_onnx_safety.py@@ -0,0 +1,191 @@+"""The shipped ONNX exporter must refuse checkpoints without safe weights, and it must+not put a graph in ``--out-dir`` before the checks that gate it have passed."""++import importlib.util+import io+import os+import sys+import tempfile+import types+from contextlib import redirect_stdout+from pathlib import Path++root = Path(__file__).resolve().parents[1]+spec = importlib.util.spec_from_file_location(+ "laya_ts_export_onnx", root / "laya-ts" / "scripts" / "export_onnx.py"+)+exporter = importlib.util.module_from_spec(spec)+spec.loader.exec_module(exporter)+++with tempfile.TemporaryDirectory() as directory:+ model_dir = Path(directory)+ (model_dir / "pytorch_model.bin").write_bytes(b"legacy weights must never be loaded")+ try:+ exporter._checkpoint_weights_path(str(model_dir))+ except SystemExit as error:+ assert "model.safetensors" in str(error)+ else:+ raise AssertionError("a legacy .bin checkpoint must be rejected")++ weights = model_dir / "model.safetensors"+ weights.touch()+ assert exporter._checkpoint_weights_path(str(model_dir)) == str(weights)+++class _TracedModule:+ """Base for the encoder/head wrappers ``main`` defines: it only calls ``eval`` on them."""++ def eval(self):+ return self+++class _Model:+ """The built network, reduced to what ``main`` touches: five submodules and ``eval``."""++ encoder = head = type_emb = scorer = act_head = None++ def load_state_dict(self, state, strict=False):+ pass++ def eval(self):+ return self++ def float(self):+ return self+++def _checkpoint(directory):+ """Write the smallest checkpoint dir ``main`` accepts: safe weights, config, tokenizer."""+ model_dir = Path(directory)+ (model_dir / "model.safetensors").write_bytes(b"weights")+ (model_dir / "rl_agent_config.json").write_text('{"decoder": {"local_attention": 128}}',+ encoding="utf-8")+ (model_dir / "tokenizer.json").write_text('{"vocab": {}}', encoding="utf-8")+ return str(model_dir)+++def _fake_torch():+ """A ``torch`` module carrying only what ``main`` calls, with tensor work stubbed out."""+ torch = types.ModuleType("torch")+ torch.long = "int64"+ torch.ones = lambda *shape, dtype=None: shape+ for name in ("nn", "onnx", "export"):+ setattr(torch, name, types.ModuleType("torch." + name))+ torch.nn.Module = _TracedModule+ torch.export.Dim = lambda name, min=None, max=None: (name, min, max)+ return torch+++def _fake_safetensors():+ root = types.ModuleType("safetensors")+ root.torch = types.ModuleType("safetensors.torch")+ root.torch.load_file = lambda path: {}+ return root+++def _export(model_dir, out_dir, events, *, export_fails=False, verify_fails=False, extra=()):+ """Drive ``main`` against faked torch and read-back seams; return its stdout.++ ``events`` records ``(stage, graph_path, out_dir_existed)`` in call order for every+ trace and every verification, which is what pins the ordering: while a stage is+ recorded, ``--out-dir`` must not exist yet and the path must not be inside it.+ ``SystemExit`` from the exporter propagates, leaving the events it reached.+ """+ torch = _fake_torch()++ def export(module, inputs, path, **kwargs):+ events.append(("trace", path, os.path.isdir(out_dir)))+ if export_fails:+ raise SystemExit("tracing failed")+ with open(path, "wb") as handle:+ handle.write(b"graph")++ def verify(enc_path, head_path, *args, **kwargs):+ events.append(("verify", (enc_path, head_path), os.path.isdir(out_dir)))+ if verify_fails:+ raise SystemExit("verification failed: logits max abs diff 2.0e-04 > 1e-4")++ def run_ref(model, head, seq, batch=1):+ return ("hidden", "logits", "act", "pos", "mask", "qtype", "att")++ torch.onnx.export = export+ safetensors = _fake_safetensors()+ names = ("torch", "torch.nn", "torch.onnx", "torch.export",+ "safetensors", "safetensors.torch")+ saved = {name: sys.modules.get(name) for name in names}+ patched = (exporter.build_model, exporter._run_ref, exporter._verify)+ printed = io.StringIO()+ try:+ sys.modules.update({"torch": torch, "torch.nn": torch.nn, "torch.onnx": torch.onnx,+ "torch.export": torch.export, "safetensors": safetensors,+ "safetensors.torch": safetensors.torch})+ exporter.build_model = lambda cfg, encoder_dir=None, pretrained=False: _Model()+ exporter._run_ref = run_ref+ exporter._verify = verify+ with redirect_stdout(printed):+ exporter.main(["--model-dir", model_dir, "--out-dir", out_dir,+ "--seq-len", "16", "--verify-len", "512", *extra])+ finally:+ exporter.build_model, exporter._run_ref, exporter._verify = patched+ for name, module in saved.items():+ if module is None:+ sys.modules.pop(name, None)+ else:+ sys.modules[name] = module+ return printed.getvalue()+++with tempfile.TemporaryDirectory() as directory:+ model_dir = _checkpoint(directory)++ # A run whose 1e-4 check fails must leave no graph behind for the Node/Web providers to+ # load, and must not even create --out-dir: the old code traced straight into it.+ failed = os.path.join(directory, "failed")+ events = []+ try:+ _export(model_dir, failed, events, verify_fails=True)+ raise AssertionError("a failed verification must abort the export")+ except SystemExit as error:+ assert "verification failed" in str(error), error+ assert [stage for stage, _, _ in events] == ["trace", "trace", "verify"], events+ assert not os.path.exists(failed), os.listdir(directory)++ # Same for a trace that dies partway: the first graph is useless without the second.+ torn = os.path.join(directory, "torn")+ events = []+ try:+ _export(model_dir, torn, events, export_fails=True)+ raise AssertionError("a failed trace must abort the export")+ except SystemExit as error:+ assert "tracing failed" in str(error), error+ assert [stage for stage, _, _ in events] == ["trace"], events+ assert not os.path.exists(torn), os.listdir(directory)++ # The passing run: every stage reads a staged graph while --out-dir is still absent,+ # then both graphs land in it. The staging dir is the temp dir the exporter made.+ ok = os.path.join(directory, "model")+ events = []+ printed = _export(model_dir, ok, events)+ assert [stage for stage, _, _ in events] == ["trace", "trace", "verify", "verify", "verify"], events+ assert all(existed is False for _, _, existed in events), events+ staged = os.path.dirname(events[0][1])+ assert staged != ok and os.path.basename(staged).startswith("laya-onnx-"), staged+ assert not os.path.exists(staged), "the staging dir must be cleaned up"+ assert events[2][1] == (os.path.join(staged, "encoder.onnx"),+ os.path.join(staged, "head.onnx")), events[2]+ assert sorted(os.listdir(ok)) == ["encoder.onnx", "head.onnx", "rl_agent_config.json",+ "tokenizer.json"], os.listdir(ok)+ for name in ("encoder.onnx", "head.onnx"):+ with open(os.path.join(ok, name), "rb") as handle:+ assert handle.read() == b"graph", name+ assert printed == ("wrote %s and %s\n" % (os.path.join(ok, "encoder.onnx"),+ os.path.join(ok, "head.onnx"))), printed++ # ``--no-verify`` still ships: it skips the checks, not the move.+ skipped = os.path.join(directory, "no-verify")+ events = []+ _export(model_dir, skipped, events, extra=("--no-verify",))+ assert [stage for stage, _, _ in events] == ["trace", "trace"], events+ assert sorted(os.listdir(skipped)) == ["encoder.onnx", "head.onnx", "rl_agent_config.json",+ "tokenizer.json"], os.listdir(skipped)