Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): verify the ONNX graphs before anything lands in --out-dir

Aashish58f4b16e8f10
diff --git a/laya-ts/scripts/export_onnx.py b/laya-ts/scripts/export_onnx.py
new file mode 100644
index 00000000..85c57efc
--- /dev/null
+++ b/laya-ts/scripts/export_onnx.py
@@ -0,0 +1,263 @@
+"""Export a Laya checkpoint to split ONNX (encoder.onnx + head.onnx).
+
+Source of truth stays ``model.safetensors``; this script produces the two ONNX
+files the ``laya-ts`` Node/Web providers load. Run once per checkpoint:
+ python laya-ts/scripts/export_onnx.py --repo convaiinnovations/laya --out-dir ./model
+ python laya-ts/scripts/export_onnx.py --repo convaiinnovations/laya --subfolder multilingual --out-dir ./model-ml
+ python laya-ts/scripts/export_onnx.py --model-dir <local ckpt> --out-dir ./model
+
+With ``--repo`` the checkpoint is downloaded from Hugging Face (safetensors +
+tokenizer + encoder) into the HF cache first, then converted. With
+``--model-dir`` an already-downloaded directory is used directly.
+
+The encoder graph takes ``(input_ids, attention_mask)`` and returns
+``last_hidden_state``. The head graph takes ``(hidden_states, marker_pos,
+marker_mask, qtype)`` and returns ``(logits, act_logits)``. Verification runs
+a torch forward at ``--seq-len`` and at ``--verify-len`` (default 512, past the
+local_attention=128 sliding window) and requires the ONNX outputs to match
+within 1e-4 at both lengths before anything is written.
+"""
+import argparse
+import json
+import os
+import shutil
+import tempfile
+
+try:
+ from laya.common import build_model
+except ImportError: # --help must work without torch installed
+ build_model = None
+
+DEFAULT_REPO = "convaiinnovations/laya"
+
+
+def _download(repo, subfolder=None, token=None):
+ """Download a checkpoint from HF into the HF cache; return the checkpoint dir."""
+ # Windows without Developer Mode can't symlink the HF cache; the hub prints
+ # a long warning and falls back to copies. Silence it, copies still work.
+ os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+ try:
+ from huggingface_hub import snapshot_download
+ except ImportError:
+ raise SystemExit("download needs huggingface_hub: pip install huggingface_hub")
+ prefix = ("%s/" % subfolder) if subfolder else ""
+ allow = [prefix + name for name in (
+ "rl_agent_config.json", "model.safetensors", "config.json",
+ "tokenizer.json", "tokenizer/*", "encoder/*",
+ )]
+ root = snapshot_download(repo, allow_patterns=allow,
+ token=token or os.environ.get("HF_TOKEN"))
+ path = os.path.join(root, subfolder) if subfolder else root
+ if not os.path.isdir(path):
+ raise SystemExit("subfolder %r not found in %r" % (subfolder, repo))
+ return path
+
+
+def _ensure_tokenizer_json(model_dir):
+ """Copies tokenizer/tokenizer.json up to the checkpoint root when needed."""
+ dest = os.path.join(model_dir, "tokenizer.json")
+ if not os.path.exists(dest):
+ src = os.path.join(model_dir, "tokenizer", "tokenizer.json")
+ if os.path.exists(src):
+ shutil.copy(src, dest)
+
+
+def _checkpoint_weights_path(model_dir):
+ """Require the checkpoint's safetensors weights before building a model."""
+ path = os.path.join(model_dir, "model.safetensors")
+ if not os.path.isfile(path):
+ raise SystemExit("checkpoint is missing model.safetensors: %s" % model_dir)
+ return path
+
+
+def parse_args(argv=None):
+ p = argparse.ArgumentParser(description=__doc__)
+ src = p.add_mutually_exclusive_group(required=True)
+ src.add_argument("--repo", help="HF repo to download first (default %s with --model-dir absent)." % DEFAULT_REPO)
+ src.add_argument("--model-dir", help="Local checkpoint dir with rl_agent_config.json (+ tokenizer.json).")
+ p.add_argument("--subfolder", default=None, help="Subfolder inside the repo (e.g. multilingual, typed-decisions).")
+ p.add_argument("--token", default=None, help="HF token (or HF_TOKEN env).")
+ p.add_argument("--out-dir", required=True, help="Output dir for encoder.onnx, head.onnx, tokenizer.json, rl_agent_config.json.")
+ p.add_argument("--opset", type=int, default=18, help="ONNX opset (default 18; the exporter implements 18 natively and a downgrade to 17 emits invalid Split nodes).")
+ p.add_argument("--seq-len", type=int, default=16, help="Dummy sequence length for tracing/verification (default 16).")
+ p.add_argument("--verify-len", type=int, default=512, help="Second length for the torch-vs-ONNX 1e-4 check (default 512; exercises sliding-window layers, local_attention=128).")
+ p.add_argument("--no-verify", action="store_true", help="Skip the torch-vs-ONNX 1e-4 check.")
+ return p.parse_args(argv)
+
+
+def main(argv=None):
+ args = parse_args(argv)
+ if build_model is None:
+ raise SystemExit("export needs torch + laya installed: pip install torch transformers onnx onnxruntime")
+ model_dir = args.model_dir or _download(args.repo or DEFAULT_REPO, args.subfolder, args.token)
+ weights_path = _checkpoint_weights_path(model_dir)
+ _ensure_tokenizer_json(model_dir)
+ import torch
+
+ class _EncoderOnly(torch.nn.Module):
+ def __init__(self, model):
+ super().__init__()
+ self.encoder = model.encoder
+
+ def forward(self, input_ids, attention_mask):
+ return self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+
+ class _HeadOnly(torch.nn.Module):
+ """Decision head tail: type_emb + transformer head + scorer + act_head."""
+
+ def __init__(self, model):
+ super().__init__()
+ self.head = model.head
+ self.type_emb = model.type_emb
+ self.scorer = model.scorer
+ self.act_head = model.act_head
+
+ def forward(self, hidden_states, marker_pos, marker_mask, qtype, attention_mask):
+ h = hidden_states + self.type_emb(qtype.squeeze(-1))[:, None, :]
+ if self.head is not None:
+ # Same as DecisionModel.forward: padding must not be attended.
+ # Without this, batch mates of unequal length corrupt each
+ # other's markers through the head transformer.
+ pad = attention_mask == 0
+ for layer in self.head.layers:
+ h = layer(h, src_key_padding_mask=pad)
+ idx = marker_pos.clamp(min=0)[:, :, None].expand(-1, -1, h.size(-1))
+ m = torch.gather(h, 1, idx)
+ logits = self.scorer(m).squeeze(-1).float()
+ logits = logits.masked_fill(~marker_mask, -1e4)
+ p = torch.softmax(logits.detach(), -1)
+ k = marker_mask.sum(-1).clamp(min=2).float()
+ ent = -(p * torch.log(p.clamp_min(1e-9))).sum(-1) / torch.log(k)
+ top2 = p.topk(2, -1).values
+ feats = torch.stack([top2[:, 0], top2[:, 0] - top2[:, 1], ent, k / 255.0], -1)
+ act_logits = self.act_head(torch.cat([h[:, 0].float(), feats], -1))
+ return logits, act_logits
+
+ with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f:
+ cfg = json.load(f)
+ # The encoder arch lives in <ckpt>/encoder/config.json (same as Agent:
+ # enc_dir = model_dir/"encoder"); the checkpoint root itself has no
+ # config.json, so passing model_dir here makes AutoConfig fail with
+ # "Unrecognized model ... no model_type key".
+ enc_dir = os.path.join(model_dir, "encoder")
+ model = build_model(cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None,
+ pretrained=False)
+ from safetensors.torch import load_file as _load_safetensors
+
+ model.load_state_dict(_load_safetensors(weights_path), strict=False)
+ model.eval().float()
+
+ # Batch must stay symbolic: tracing with batch=1 lets dynamo bake batch=1
+ # into a head reshape (batch=2 then fails with "204 by 408" broadcast).
+ # Explicit Dims + batch=2 dummies keep it dynamic; verify covers 1 and 2.
+ batch_dim = torch.export.Dim("batch", min=1, max=64)
+ seq_dim = torch.export.Dim("seq", min=1, max=8192)
+ markers_dim = torch.export.Dim("markers", min=1, max=256)
+ enc_shapes = ({0: batch_dim, 1: seq_dim}, {0: batch_dim, 1: seq_dim})
+ head_shapes = ({0: batch_dim, 1: seq_dim}, {0: batch_dim, 1: markers_dim},
+ {0: batch_dim, 1: markers_dim}, {0: batch_dim},
+ {0: batch_dim, 1: seq_dim})
+
+ seq = int(args.seq_len)
+ input_ids = torch.ones(2, seq, dtype=torch.long)
+ attention_mask = torch.ones(2, seq, dtype=torch.long)
+ ref_hidden, ref_logits, ref_act, marker_pos, marker_mask, qtype, ref_att = _run_ref(model, _HeadOnly(model).eval(), seq, batch=2)
+
+ # The module docstring promises the outputs match "within 1e-4 at both lengths before anything is
+ # written", and verification reads the graphs back off disk, so "written" has to mean "written to
+ # --out-dir". Tracing straight into it left two unverified graphs there on a failed run -- which is
+ # exactly what the Node/Web providers then load -- and created --out-dir even when the run refused.
+ staged = tempfile.mkdtemp(prefix="laya-onnx-")
+ try:
+ enc_path = os.path.join(staged, "encoder.onnx")
+ head_path = os.path.join(staged, "head.onnx")
+
+ torch.onnx.export(
+ _EncoderOnly(model).eval(),
+ (input_ids, attention_mask),
+ enc_path,
+ input_names=["input_ids", "attention_mask"],
+ output_names=["last_hidden_state"],
+ dynamic_axes={"input_ids": {0: "batch", 1: "seq"}, "attention_mask": {0: "batch", 1: "seq"}, "last_hidden_state": {0: "batch", 1: "seq"}},
+ dynamic_shapes=enc_shapes,
+ opset_version=args.opset,
+ )
+ torch.onnx.export(
+ _HeadOnly(model).eval(),
+ (ref_hidden, marker_pos, marker_mask, qtype, ref_att),
+ head_path,
+ input_names=["hidden_states", "marker_pos", "marker_mask", "qtype", "attention_mask"],
+ output_names=["logits", "act_logits"],
+ dynamic_axes={"hidden_states": {0: "batch", 1: "seq"}, "marker_pos": {0: "batch", 1: "markers"}, "marker_mask": {0: "batch", 1: "markers"}, "qtype": {0: "batch"}},
+ dynamic_shapes=head_shapes,
+ opset_version=args.opset,
+ )
+
+ if not args.no_verify:
+ _verify(enc_path, head_path, input_ids, attention_mask, marker_pos, marker_mask, qtype, ref_att, ref_logits, ref_act)
+ b1 = _run_ref(model, _HeadOnly(model).eval(), seq, batch=1)
+ _verify(enc_path, head_path, torch.ones(1, seq, dtype=torch.long),
+ torch.ones(1, seq, dtype=torch.long), *b1[3:6], b1[6], b1[1], b1[2])
+ if int(args.verify_len) != seq:
+ v_hidden, v_logits, v_act, v_pos, v_mask, v_qtype, v_att2 = _run_ref(model, _HeadOnly(model).eval(), int(args.verify_len), batch=2)
+ v_ids = torch.ones(2, int(args.verify_len), dtype=torch.long)
+ v_att = torch.ones(2, int(args.verify_len), dtype=torch.long)
+ _verify(enc_path, head_path, v_ids, v_att, v_pos, v_mask, v_qtype, v_att2, v_logits, v_act)
+
+ os.makedirs(args.out_dir, exist_ok=True)
+ for name in ("encoder.onnx", "head.onnx"):
+ shutil.move(os.path.join(staged, name), os.path.join(args.out_dir, name))
+ for name in ("tokenizer.json", "rl_agent_config.json"):
+ src = os.path.join(model_dir, name)
+ if os.path.exists(src):
+ shutil.copy(src, os.path.join(args.out_dir, name))
+ print("wrote %s and %s" % (os.path.join(args.out_dir, "encoder.onnx"),
+ os.path.join(args.out_dir, "head.onnx")))
+ finally:
+ shutil.rmtree(staged, ignore_errors=True)
+
+
+def _run_ref(model, head, seq, batch=1):
+ import torch
+
+ input_ids = torch.ones(batch, seq, dtype=torch.long)
+ attention_mask = torch.ones(batch, seq, dtype=torch.long)
+ with torch.inference_mode():
+ ref_hidden = model.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+ marker_pos = torch.tensor([[1, 2]] * batch, dtype=torch.long)
+ marker_mask = torch.tensor([[True, True]] * batch)
+ qtype = torch.tensor([[0]] * batch, dtype=torch.long)
+ ref_logits, ref_act = head(ref_hidden, marker_pos, marker_mask, qtype, attention_mask)
+ return ref_hidden, ref_logits, ref_act, marker_pos, marker_mask, qtype, attention_mask
+
+
+def _verify(enc_path, head_path, input_ids, attention_mask, marker_pos, marker_mask, qtype, head_att, ref_logits, ref_act):
+ import numpy as np
+
+ try:
+ import onnxruntime as ort
+ except ImportError:
+ raise SystemExit("export --no-verify not given but onnxruntime is missing: pip install onnxruntime")
+ enc = ort.InferenceSession(enc_path, providers=["CPUExecutionProvider"])
+ head = ort.InferenceSession(head_path, providers=["CPUExecutionProvider"])
+ (hidden,) = enc.run(None, {"input_ids": input_ids.numpy(), "attention_mask": attention_mask.numpy()})
+ logits, act = head.run(None, {"hidden_states": hidden.astype(np.float32), "marker_pos": marker_pos.numpy(), "marker_mask": marker_mask.numpy(), "qtype": qtype.numpy(), "attention_mask": head_att.numpy()})
+ diff = float(np.abs(logits - ref_logits.detach().numpy()).max())
+ if diff > 1e-4:
+ raise SystemExit("verification failed: logits max abs diff %.2e > 1e-4" % diff)
+ # act_logits are consumed only through softmax (rounded to 4dp downstream),
+ # and dynamo decompositions add ~5e-4 raw-logit noise that softmax washes
+ # out. Compare the probabilities TS actually reads instead of raw logits.
+ got_p = np.exp(act - act.max(axis=-1, keepdims=True))
+ got_p /= got_p.sum(axis=-1, keepdims=True)
+ ref = ref_act.detach().numpy()
+ ref_p = np.exp(ref - ref.max(axis=-1, keepdims=True))
+ ref_p /= ref_p.sum(axis=-1, keepdims=True)
+ pdiff = float(np.abs(got_p - ref_p).max())
+ if pdiff > 1e-4:
+ raise SystemExit("verification failed: act_probs max abs diff %.2e > 1e-4" % pdiff)
+ print("verify ok: torch vs onnx match within 1e-4 (logits %.1e, act_probs %.1e)" % (diff, pdiff))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_export_onnx_safety.py b/tests/test_export_onnx_safety.py
new file mode 100644
index 00000000..5154d636
--- /dev/null
+++ b/tests/test_export_onnx_safety.py
@@ -0,0 +1,191 @@
+"""The shipped ONNX exporter must refuse checkpoints without safe weights, and it must
+not put a graph in ``--out-dir`` before the checks that gate it have passed."""
+
+import importlib.util
+import io
+import os
+import sys
+import tempfile
+import types
+from contextlib import redirect_stdout
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location(
+ "laya_ts_export_onnx", root / "laya-ts" / "scripts" / "export_onnx.py"
+)
+exporter = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(exporter)
+
+
+with tempfile.TemporaryDirectory() as directory:
+ model_dir = Path(directory)
+ (model_dir / "pytorch_model.bin").write_bytes(b"legacy weights must never be loaded")
+ try:
+ exporter._checkpoint_weights_path(str(model_dir))
+ except SystemExit as error:
+ assert "model.safetensors" in str(error)
+ else:
+ raise AssertionError("a legacy .bin checkpoint must be rejected")
+
+ weights = model_dir / "model.safetensors"
+ weights.touch()
+ assert exporter._checkpoint_weights_path(str(model_dir)) == str(weights)
+
+
+class _TracedModule:
+ """Base for the encoder/head wrappers ``main`` defines: it only calls ``eval`` on them."""
+
+ def eval(self):
+ return self
+
+
+class _Model:
+ """The built network, reduced to what ``main`` touches: five submodules and ``eval``."""
+
+ encoder = head = type_emb = scorer = act_head = None
+
+ def load_state_dict(self, state, strict=False):
+ pass
+
+ def eval(self):
+ return self
+
+ def float(self):
+ return self
+
+
+def _checkpoint(directory):
+ """Write the smallest checkpoint dir ``main`` accepts: safe weights, config, tokenizer."""
+ model_dir = Path(directory)
+ (model_dir / "model.safetensors").write_bytes(b"weights")
+ (model_dir / "rl_agent_config.json").write_text('{"decoder": {"local_attention": 128}}',
+ encoding="utf-8")
+ (model_dir / "tokenizer.json").write_text('{"vocab": {}}', encoding="utf-8")
+ return str(model_dir)
+
+
+def _fake_torch():
+ """A ``torch`` module carrying only what ``main`` calls, with tensor work stubbed out."""
+ torch = types.ModuleType("torch")
+ torch.long = "int64"
+ torch.ones = lambda *shape, dtype=None: shape
+ for name in ("nn", "onnx", "export"):
+ setattr(torch, name, types.ModuleType("torch." + name))
+ torch.nn.Module = _TracedModule
+ torch.export.Dim = lambda name, min=None, max=None: (name, min, max)
+ return torch
+
+
+def _fake_safetensors():
+ root = types.ModuleType("safetensors")
+ root.torch = types.ModuleType("safetensors.torch")
+ root.torch.load_file = lambda path: {}
+ return root
+
+
+def _export(model_dir, out_dir, events, *, export_fails=False, verify_fails=False, extra=()):
+ """Drive ``main`` against faked torch and read-back seams; return its stdout.
+
+ ``events`` records ``(stage, graph_path, out_dir_existed)`` in call order for every
+ trace and every verification, which is what pins the ordering: while a stage is
+ recorded, ``--out-dir`` must not exist yet and the path must not be inside it.
+ ``SystemExit`` from the exporter propagates, leaving the events it reached.
+ """
+ torch = _fake_torch()
+
+ def export(module, inputs, path, **kwargs):
+ events.append(("trace", path, os.path.isdir(out_dir)))
+ if export_fails:
+ raise SystemExit("tracing failed")
+ with open(path, "wb") as handle:
+ handle.write(b"graph")
+
+ def verify(enc_path, head_path, *args, **kwargs):
+ events.append(("verify", (enc_path, head_path), os.path.isdir(out_dir)))
+ if verify_fails:
+ raise SystemExit("verification failed: logits max abs diff 2.0e-04 > 1e-4")
+
+ def run_ref(model, head, seq, batch=1):
+ return ("hidden", "logits", "act", "pos", "mask", "qtype", "att")
+
+ torch.onnx.export = export
+ safetensors = _fake_safetensors()
+ names = ("torch", "torch.nn", "torch.onnx", "torch.export",
+ "safetensors", "safetensors.torch")
+ saved = {name: sys.modules.get(name) for name in names}
+ patched = (exporter.build_model, exporter._run_ref, exporter._verify)
+ printed = io.StringIO()
+ try:
+ sys.modules.update({"torch": torch, "torch.nn": torch.nn, "torch.onnx": torch.onnx,
+ "torch.export": torch.export, "safetensors": safetensors,
+ "safetensors.torch": safetensors.torch})
+ exporter.build_model = lambda cfg, encoder_dir=None, pretrained=False: _Model()
+ exporter._run_ref = run_ref
+ exporter._verify = verify
+ with redirect_stdout(printed):
+ exporter.main(["--model-dir", model_dir, "--out-dir", out_dir,
+ "--seq-len", "16", "--verify-len", "512", *extra])
+ finally:
+ exporter.build_model, exporter._run_ref, exporter._verify = patched
+ for name, module in saved.items():
+ if module is None:
+ sys.modules.pop(name, None)
+ else:
+ sys.modules[name] = module
+ return printed.getvalue()
+
+
+with tempfile.TemporaryDirectory() as directory:
+ model_dir = _checkpoint(directory)
+
+ # A run whose 1e-4 check fails must leave no graph behind for the Node/Web providers to
+ # load, and must not even create --out-dir: the old code traced straight into it.
+ failed = os.path.join(directory, "failed")
+ events = []
+ try:
+ _export(model_dir, failed, events, verify_fails=True)
+ raise AssertionError("a failed verification must abort the export")
+ except SystemExit as error:
+ assert "verification failed" in str(error), error
+ assert [stage for stage, _, _ in events] == ["trace", "trace", "verify"], events
+ assert not os.path.exists(failed), os.listdir(directory)
+
+ # Same for a trace that dies partway: the first graph is useless without the second.
+ torn = os.path.join(directory, "torn")
+ events = []
+ try:
+ _export(model_dir, torn, events, export_fails=True)
+ raise AssertionError("a failed trace must abort the export")
+ except SystemExit as error:
+ assert "tracing failed" in str(error), error
+ assert [stage for stage, _, _ in events] == ["trace"], events
+ assert not os.path.exists(torn), os.listdir(directory)
+
+ # The passing run: every stage reads a staged graph while --out-dir is still absent,
+ # then both graphs land in it. The staging dir is the temp dir the exporter made.
+ ok = os.path.join(directory, "model")
+ events = []
+ printed = _export(model_dir, ok, events)
+ assert [stage for stage, _, _ in events] == ["trace", "trace", "verify", "verify", "verify"], events
+ assert all(existed is False for _, _, existed in events), events
+ staged = os.path.dirname(events[0][1])
+ assert staged != ok and os.path.basename(staged).startswith("laya-onnx-"), staged
+ assert not os.path.exists(staged), "the staging dir must be cleaned up"
+ assert events[2][1] == (os.path.join(staged, "encoder.onnx"),
+ os.path.join(staged, "head.onnx")), events[2]
+ assert sorted(os.listdir(ok)) == ["encoder.onnx", "head.onnx", "rl_agent_config.json",
+ "tokenizer.json"], os.listdir(ok)
+ for name in ("encoder.onnx", "head.onnx"):
+ with open(os.path.join(ok, name), "rb") as handle:
+ assert handle.read() == b"graph", name
+ assert printed == ("wrote %s and %s\n" % (os.path.join(ok, "encoder.onnx"),
+ os.path.join(ok, "head.onnx"))), printed
+
+ # ``--no-verify`` still ships: it skips the checks, not the move.
+ skipped = os.path.join(directory, "no-verify")
+ events = []
+ _export(model_dir, skipped, events, extra=("--no-verify",))
+ assert [stage for stage, _, _ in events] == ["trace", "trace"], events
+ assert sorted(os.listdir(skipped)) == ["encoder.onnx", "head.onnx", "rl_agent_config.json",
+ "tokenizer.json"], os.listdir(skipped)