Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(dotnet): add benchmark sample with Python baseline
kevin-gatimu9f5c6bdffc80
diff --git a/laya-dotnet/samples/Laya.Benchmark/.gitignore b/laya-dotnet/samples/Laya.Benchmark/.gitignorenew file mode 100644index 00000000..9b08c2d8--- /dev/null+++ b/laya-dotnet/samples/Laya.Benchmark/.gitignore@@ -0,0 +1,9 @@+[Bb]in/+[Oo]bj/+*.user+BenchmarkDotNet.Artifacts/++# Downloaded model artifacts / run output+*.onnx+*.onnx_data+*.csvdiff --git a/laya-dotnet/samples/Laya.Benchmark/Laya.Benchmark.csproj b/laya-dotnet/samples/Laya.Benchmark/Laya.Benchmark.csprojnew file mode 100644index 00000000..52ccdb29--- /dev/null+++ b/laya-dotnet/samples/Laya.Benchmark/Laya.Benchmark.csproj@@ -0,0 +1,16 @@+<Project Sdk="Microsoft.NET.Sdk">++ <ItemGroup>+ <ProjectReference Include="..\..\src\Laya\Laya.csproj" />+ </ItemGroup>++ <PropertyGroup>+ <OutputType>Exe</OutputType>+ <TargetFramework>net10.0</TargetFramework>+ <ImplicitUsings>enable</ImplicitUsings>+ <Nullable>enable</Nullable>+ <!-- Benchmark must run Release for meaningful numbers. -->+ <Optimize>true</Optimize>+ </PropertyGroup>++</Project>diff --git a/laya-dotnet/samples/Laya.Benchmark/Program.cs b/laya-dotnet/samples/Laya.Benchmark/Program.csnew file mode 100644index 00000000..777e5639--- /dev/null+++ b/laya-dotnet/samples/Laya.Benchmark/Program.cs@@ -0,0 +1,232 @@+// Benchmark the Laya .NET SDK across one checkpoint.+//+// Measures:+// cold latency : engine creation + first Predict (ms)+// warm latency : median and p95 over N_WARM timed calls (ms), after N_WARMUP warmups+// peak working set (MB), via Process.WorkingSet64+// answer snapshot for each bench case+//+// Outputs one JSON object to stdout.+//+// Usage:+// dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint multilingual+// dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint english+// dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint typed-decisions++using System.Diagnostics;+using System.Text.Json;+using System.Text.Json.Serialization;+using Laya;++const int N_WARMUP = 5;+const int N_WARM = 30;++// ── parse args ────────────────────────────────────────────────────────────────+LayaCheckpoint checkpoint = LayaCheckpoint.Multilingual;+string? modelDirectory = null;+var onnxRoot = Environment.GetEnvironmentVariable("LAYA_ONNX_ROOT");++for (var i = 0; i < args.Length; i++)+{+ switch (args[i])+ {+ case "--checkpoint":+ if (i + 1 >= args.Length) Die("--checkpoint needs a value.");+ checkpoint = args[++i] switch+ {+ "multilingual" => LayaCheckpoint.Multilingual,+ "english" => LayaCheckpoint.English,+ "typed-decisions" => LayaCheckpoint.TypedDecisions,+ var s => DieWith<LayaCheckpoint>($"Unknown checkpoint: {s}"),+ };+ break;+ case "--model-dir":+ if (i + 1 >= args.Length) Die("--model-dir needs a value.");+ modelDirectory = args[++i];+ break;+ default:+ Die($"Unrecognised argument: {args[i]}");+ break;+ }+}++var checkpointName = checkpoint switch+{+ LayaCheckpoint.Multilingual => "multilingual",+ LayaCheckpoint.English => "english",+ LayaCheckpoint.TypedDecisions => "typed-decisions",+ _ => throw new InvalidOperationException(),+};++// If LAYA_ONNX_ROOT is set and no explicit --model-dir was given, derive the dir.+if (modelDirectory is null && onnxRoot is not null)+ modelDirectory = Path.Combine(onnxRoot, checkpointName);++// ── bench cases ───────────────────────────────────────────────────────────────+// Mirrors laya-dotnet/tools/bench_python.py: the same two states and question set.+var englishState = new Dictionary<string, object?>+{+ ["from"] = "[email protected]",+ ["subject"] = "Duplicate charge on invoice #4411",+ ["body"] = "Hi, we were billed twice for March. Please refund the duplicate today "+ + "or we will cancel our plan.",+};+var hindiState = new Dictionary<string, object?>+{+ ["body"] = "मार्च का बिल "+ + "दो बार लिया गया, "+ + "कृपया रिफंड करें।",+};+var questions = new QuestionSet+{+ ["department"] = Question.Choice(+ "Which department should handle this request?",+ ("billing", "invoices, payments, refunds"),+ ("technical", "bugs, outages, system errors"),+ ("sales", "pricing, new contracts"),+ ("other", "everything else")),+ ["urgency"] = Question.Score(+ "How urgent is this request?",+ "not urgent", "soon", "critical deadline or blocking issue"),+ ["churn_risk"] = Question.Noul("Does the user threaten to cancel or leave?"),+ ["refund_requested"] = Question.Noul("Does the user explicitly request a refund?"),+};+var benchCases = new (string Name, object? State)[]+{+ ("sample_app_english", englishState),+ ("sample_app_hindi", hindiState),+};++// ── cold measurement ──────────────────────────────────────────────────────────+var coldClock = Stopwatch.StartNew();+LayaEngine engine;+try+{+ engine = LayaEngine.Create(new LayaOptions+ {+ Checkpoint = checkpoint,+ ModelDirectory = modelDirectory,+ });+}+catch (Exception ex) when (ex is DirectoryNotFoundException or FileNotFoundException)+{+ Console.Error.WriteLine(ex.Message);+ return 1;+}++// First call (still part of cold, as ORT JITs on the first run).+var (firstName, firstState) = benchCases[0];+engine.Predict(firstState, questions);+coldClock.Stop();+var coldMs = coldClock.Elapsed.TotalMilliseconds;++Console.Error.WriteLine($"[bench_dotnet] cold={coldMs:F0} ms, running warm-up...");++// ── warmup ────────────────────────────────────────────────────────────────────+var callIdx = 0;+(object? State, QuestionSet Qs) NextCall()+{+ var c = benchCases[callIdx++ % benchCases.Length];+ return (c.State, questions);+}++for (var i = 0; i < N_WARMUP; i++)+{+ var (s, q) = NextCall();+ engine.Predict(s, q);+}++// ── warm measurement ──────────────────────────────────────────────────────────+var timesMs = new double[N_WARM];+for (var i = 0; i < N_WARM; i++)+{+ var (s, q) = NextCall();+ var t0 = Stopwatch.GetTimestamp();+ engine.Predict(s, q);+ timesMs[i] = Stopwatch.GetElapsedTime(t0).TotalMilliseconds;+}+Array.Sort(timesMs);+var medianMs = timesMs[timesMs.Length / 2];+var p95Ms = timesMs[(int)(timesMs.Length * 0.95)];++// ── peak working set ──────────────────────────────────────────────────────────+var peakMb = Process.GetCurrentProcess().PeakWorkingSet64 / (1024.0 * 1024.0);++Console.Error.WriteLine(+ $"[bench_dotnet] warm median={medianMs:F1} ms p95={p95Ms:F1} ms peak={peakMb:F0} MB");++// ── answer snapshot ───────────────────────────────────────────────────────────+var answers = new Dictionary<string, object>();+foreach (var (name, state) in benchCases)+{+ var result = engine.Predict(state, questions);+ var caseAnswers = new Dictionary<string, object>();+ foreach (var id in result.Ids)+ {+ var a = result[id];+ switch (a)+ {+ case ChoiceAnswer choice:+ caseAnswers[id] = new+ {+ type = "choice",+ choice = choice.Choice,+ probabilities = choice.Probabilities.ToDictionary(p => p.Key, p => p.Value),+ confidence = choice.Confidence,+ act_probability = choice.Action.ActProbability,+ };+ break;+ case ScoreAnswer score:+ caseAnswers[id] = new+ {+ type = "score",+ score = score.Score,+ probabilities = score.Probabilities.Select((p, i) => (i.ToString(), p))+ .ToDictionary(x => x.Item1, x => x.p),+ confidence = score.Confidence,+ act_probability = score.Action.ActProbability,+ };+ break;+ case NoulAnswer noul:+ caseAnswers[id] = new+ {+ type = "noul",+ noul = noul.Probability,+ confidence = noul.Confidence,+ act_probability = noul.Action.ActProbability,+ };+ break;+ }+ }+ answers[name] = caseAnswers;+}++engine.Dispose();++// ── emit JSON ─────────────────────────────────────────────────────────────────+var output = new+{+ checkpoint = checkpointName,+ runtime = "DotNet + ONNX Runtime",+ threads = "ORT default",+ cold_ms = Math.Round(coldMs, 1),+ warm_times_ms = timesMs.Select(t => Math.Round(t, 2)).ToArray(),+ warm_median_ms = Math.Round(medianMs, 1),+ warm_p95_ms = Math.Round(p95Ms, 1),+ n_warmup = N_WARMUP,+ n_warm = N_WARM,+ peak_wset_mb = Math.Round(peakMb, 1),+ answers,+};++var json = JsonSerializer.Serialize(output, new JsonSerializerOptions+{+ WriteIndented = true,+ DefaultIgnoreCondition = JsonIgnoreCondition.Never,+});+Console.WriteLine(json);+return 0;++// ── helpers ───────────────────────────────────────────────────────────────────+static void Die(string msg) { Console.Error.WriteLine(msg); Environment.Exit(2); }+static T DieWith<T>(string msg) { Die(msg); return default!; }diff --git a/laya-dotnet/samples/Laya.Benchmark/README.md b/laya-dotnet/samples/Laya.Benchmark/README.mdnew file mode 100644index 00000000..16e64d73--- /dev/null+++ b/laya-dotnet/samples/Laya.Benchmark/README.md@@ -0,0 +1,123 @@+# Laya.Benchmark++A small console app that measures the Laya .NET SDK on one checkpoint: how long the first call takes, how fast later calls are, and how much memory the process uses. It also records the answers, so they can be compared with the Python SDK.++It pairs with [`tools/bench_python.py`](../../tools/bench_python.py). Both use the same inputs, so their numbers and answers can be compared directly.++## What it measures++| Metric | How |+|---|---|+| Cold latency | `LayaEngine.Create` plus the first `Predict` call, in ms. This includes loading the model and ONNX Runtime's first-run setup. |+| Warm latency | Median and p95 over 30 timed `Predict` calls, after 5 warm-up calls. The calls alternate between the two cases. |+| Peak memory | `Process.PeakWorkingSet64`, in MB, read after the warm runs |+| Answers | Each question's answer for each case: choice, score or yes/no probability, plus the probabilities, confidence and act probability |++ONNX Runtime runs on the CPU with its default thread settings.++## Inputs++There are two cases, both asked the same four questions. The inputs match `laya-dotnet/tools/sample_cases.py` and the `Laya.Sample` app.++| Case | Input |+|---|---|+| `sample_app_english` | A billing email about a duplicate charge that asks for a refund and threatens to cancel |+| `sample_app_hindi` | A short Hindi message about being billed twice for March |++The four questions:+- `department`: choice of billing, technical, sales or other+- `urgency`: score from 0 to 2+- `churn_risk`: yes/no+- `refund_requested`: yes/no++## Prerequisites++- .NET 10 SDK+- An exported ONNX folder for the checkpoint you want to test. See [`MODELS.md`](../../MODELS.md) for how to export them with `laya-dotnet/tools/export_onnx.py`.++## Usage++Run commands from `laya-dotnet`, and always build with `-c Release`: a Debug build gives misleading timings.++```powershell+# Easiest: point LAYA_ONNX_ROOT at the folder that holds all three checkpoints+$env:LAYA_ONNX_ROOT = "C:/models/laya"++dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint multilingual+dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint english+dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint typed-decisions++# Or give one checkpoint's folder directly+dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint english --model-dir C:/models/laya/english+```++| Argument | Meaning |+|---|---|+| `--checkpoint` | `multilingual` (default), `english` or `typed-decisions` |+| `--model-dir` | Folder containing `model.onnx`, `model.onnx.data`, the tokenizer files and `rl_agent_config.json`. If you leave it out, the app uses `$LAYA_ONNX_ROOT/<checkpoint>` when that variable is set, and otherwise the SDK's normal resolution order. |++`--model-dir` takes one checkpoint's folder (`onnx/english`), not the export root (`onnx/`). To+point at the root, set `LAYA_ONNX_ROOT` as shown above. There is no `--model-root` flag here;+that one belongs to [`Laya.Routing`](../Laya.Routing/README.md) and is rejected with+`Unrecognised argument`.++From this sample's own directory (`laya-dotnet/samples/Laya.Benchmark`), leave out `--project`.+Keep the bare `--`: it separates `dotnet run`'s options from the app's.++```powershell+dotnet run -c Release -- --checkpoint english --model-dir C:/models/laya/english+```++Progress lines go to stderr. The results are printed to stdout as one JSON object, so you can redirect them to a file:++```powershell+dotnet run --project samples/Laya.Benchmark -c Release -- --checkpoint english > bench_dotnet_english.json+```++Exit codes:+- `0`: success+- `1`: the model files weren't found+- `2`: bad arguments++## Output++```jsonc+{+ "checkpoint": "english", // TypedDecisions is written as "typeddecisions"+ "runtime": "DotNet + ONNX Runtime",+ "threads": "ORT default",+ "cold_ms": 0.0,+ "warm_times_ms": [ /* 30 sorted timings */ ],+ "warm_median_ms": 0.0,+ "warm_p95_ms": 0.0,+ "n_warmup": 5,+ "n_warm": 30,+ "peak_wset_mb": 0.0,+ "answers": {+ "sample_app_english": {+ "department": { "type": "choice", "choice": "...", "probabilities": { }, "confidence": 0.0, "act_probability": 0.0 },+ "urgency": { "type": "score", "score": 0.0, "probabilities": { }, "confidence": 0.0, "act_probability": 0.0 },+ "churn_risk": { "type": "noul", "noul": 0.0, "confidence": 0.0, "act_probability": 0.0 },+ "refund_requested": { "type": "noul", "noul": 0.0, "confidence": 0.0, "act_probability": 0.0 }+ },+ "sample_app_hindi": { /* same shape */ }+ }+}+```++## Comparing with Python++Run the Python counterpart with the repo's virtual environment. Set `HF_HOME` if you want the PyTorch weights to download somewhere other than the default cache.++```powershell+$env:HF_HOME = "C:\hf_cache"+.venv/Scripts/python.exe laya-dotnet/tools/bench_python.py --checkpoint all --output bench_results.json+```++Python runs the PyTorch model directly on the CPU. .NET runs the exported ONNX graph through ONNX Runtime. Expect the answers to match, with tiny probability differences from floating-point rounding (the parity tests allow for this), and expect the timings to differ because the runtimes differ.++## Notes++- **Where the project sits:** it isn't part of `Laya.slnx`, so `dotnet build` / `dotnet test` on the solution skip it. It's a measurement tool, not a shipped sample.+- **Running more than one:** run one checkpoint at a time. The ModernBERT-large checkpoints (english, typed-decisions) are about 1.7 GB each on disk.+- **When to measure:** cold latency depends on disk and OS file cache. Run twice and report the second cold number if you want warm file-cache timings.diff --git a/laya-dotnet/tools/bench_python.py b/laya-dotnet/tools/bench_python.pynew file mode 100644index 00000000..737e07e1--- /dev/null+++ b/laya-dotnet/tools/bench_python.py@@ -0,0 +1,327 @@+"""Benchmark the Python laya SDK across all three checkpoints.++Measures:+ - Cold latency: model load + first Predict call (ms)+ - Warm latency: median and p95 over N_WARM timed calls (ms), after N_WARMUP warmups+ - Peak RSS/working-set (MB), via psutil if available+ - Answer snapshot for each case, for comparison with the .NET results++Outputs JSON to stdout so the caller can parse and tabulate.++By default this benchmarks the public SDK (`laya.load`, PyTorch eager on CPU). Pass+`--runtime onnx` to run the same SDK code path with the exported ONNX graph as the forward+pass, which separates runtime cost from language cost when comparing with the .NET SDK.++Usage:+ python laya-dotnet/tools/bench_python.py --checkpoint multilingual+ python laya-dotnet/tools/bench_python.py --checkpoint multilingual --runtime onnx+ python laya-dotnet/tools/bench_python.py --checkpoint english+ python laya-dotnet/tools/bench_python.py --checkpoint typed-decisions+ python laya-dotnet/tools/bench_python.py --checkpoint all # runs all three in sequence+ python laya-dotnet/tools/bench_python.py --checkpoint all --output bench_results.json+"""++import argparse+import json+import os+import sys+import time++TOOLS = os.path.dirname(os.path.abspath(__file__))+REPO = os.path.dirname(os.path.dirname(TOOLS))+sys.path.insert(0, REPO)+sys.path.insert(0, TOOLS)++from sample_cases import SAMPLE_ENGLISH_STATE, SAMPLE_HINDI_STATE, SAMPLE_QUESTIONS++N_WARMUP = 5+N_WARM = 30++# The benchmark cases: a subset of the full golden suite chosen to cover real behaviour+# without taking forever. These are the same cases the task brief specifies.+BENCH_CASES = [+ {+ "name": "sample_app_english",+ "state": SAMPLE_ENGLISH_STATE,+ "questions": SAMPLE_QUESTIONS,+ },+ {+ "name": "sample_app_hindi",+ "state": SAMPLE_HINDI_STATE,+ "questions": SAMPLE_QUESTIONS,+ },+]++ONNX_ROOT = os.path.join(REPO, "onnx")++CHECKPOINTS = ["english", "multilingual", "typed-decisions"]+++def _onnx_dir(checkpoint: str) -> str:+ return os.path.join(ONNX_ROOT, checkpoint)+++def _load_agent(onnx_dir: str):+ """Load an Agent using the ONNX forward pass, exactly as dump_golden.py does."""+ import numpy as np+ import onnxruntime as ort+ import torch+ from transformers import AutoTokenizer++ from laya.agent import Agent, _fix_tokenizer_config+ from laya.common import clamp_temperature++ NP_DTYPES = {+ "tensor(int64)": np.int64,+ "tensor(int32)": np.int32,+ "tensor(bool)": np.bool_,+ "tensor(float)": np.float32,+ }++ class OnnxForward:+ MIN_MARKERS = 2++ def __init__(self, session):+ self.sess = session+ self.dtypes = {i.name: NP_DTYPES[i.type] for i in session.get_inputs()}++ def _pad_markers(self, marker_pos, marker_mask):+ k = marker_pos.shape[1]+ if k >= self.MIN_MARKERS:+ return marker_pos, marker_mask+ pad = self.MIN_MARKERS - k+ marker_pos = torch.cat(+ [marker_pos, torch.zeros((marker_pos.shape[0], pad), dtype=marker_pos.dtype)], 1+ )+ marker_mask = torch.cat(+ [marker_mask, torch.zeros((marker_mask.shape[0], pad), dtype=torch.bool)], 1+ )+ return marker_pos, marker_mask++ def __call__(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):+ marker_pos, marker_mask = self._pad_markers(marker_pos, marker_mask)+ feeds = {+ n: t.cpu().numpy().astype(self.dtypes[n])+ for n, t in [+ ("input_ids", input_ids),+ ("attention_mask", attention_mask),+ ("marker_pos", marker_pos),+ ("marker_mask", marker_mask),+ ("qtype", qtype),+ ]+ }+ logits, act_logits = self.sess.run(["logits", "act_logits"], feeds)+ return torch.from_numpy(logits), torch.from_numpy(act_logits)++ with open(os.path.join(onnx_dir, "rl_agent_config.json"), encoding="utf-8") as f:+ cfg = json.load(f)++ _fix_tokenizer_config(onnx_dir)+ tok = AutoTokenizer.from_pretrained(os.path.join(onnx_dir, "tokenizer"))++ sess_opts = ort.SessionOptions()+ # Single-thread to match default Python behaviour: the ONNX runtime defaults to+ # using all available cores; pin to 1 so the number is reproducible.+ sess_opts.intra_op_num_threads = 1+ sess_opts.inter_op_num_threads = 1+ sess = ort.InferenceSession(+ os.path.join(onnx_dir, "model.onnx"),+ sess_options=sess_opts,+ providers=["CPUExecutionProvider"],+ )++ agent = Agent.__new__(Agent)+ agent.cfg = cfg+ agent.tok = tok+ import torch as _torch+ agent.device = _torch.device("cpu")+ agent.dtype = _torch.float32+ agent.model = OnnxForward(sess)+ agent.temperature = [clamp_temperature(t) for t in cfg.get("temperature", [1.0, 1.0, 1.0])]+ agent.temperature_by_options = {+ k: clamp_temperature(v) for k, v in cfg.get("temperature_by_options", {}).items()+ }+ return agent+++def _load_sdk_agent(checkpoint: str):+ """Load the checkpoint through the public Python SDK (PyTorch eager, CPU)."""+ import laya++ subfolder = None if checkpoint == "english" else checkpoint+ return laya.load("convaiinnovations/laya", device="cpu", subfolder=subfolder)+++def _peak_wset_mb():+ """Return the process peak working set (Windows) or peak RSS (other) in MB."""+ try:+ import psutil+ except ImportError:+ return None+ mi = psutil.Process(os.getpid()).memory_info()+ peak = getattr(mi, "peak_wset", None)+ if peak is None:+ import resource # POSIX: ru_maxrss is KiB on Linux+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024+ return round(peak / 1024 / 1024, 1)+++def bench_checkpoint(checkpoint: str, runtime: str) -> dict:+ """Run the full benchmark for one checkpoint. Returns a result dict."""+ print(f"[bench_python] {checkpoint} ({runtime}): loading...", flush=True)++ onnx_dir = _onnx_dir(checkpoint)+ if runtime == "onnx" and not os.path.isdir(onnx_dir):+ return {"checkpoint": checkpoint, "error": f"ONNX dir not found: {onnx_dir}"}++ # Imports are timed separately so cold latency matches the .NET number+ # (engine creation + first call, excluding runtime start-up).+ t_import = time.perf_counter()+ import torch # noqa: F401+ import laya # noqa: F401+ import_ms = (time.perf_counter() - t_import) * 1000.0++ # ── cold latency ─────────────────────────────────────────────────────────+ # Measure load + first predict together as the "cold" number.+ t_cold_start = time.perf_counter()+ agent = _load_sdk_agent(checkpoint) if runtime == "torch" else _load_agent(onnx_dir)++ # Use the first benchmark case for the cold call.+ first_case = BENCH_CASES[0]+ _ = agent.system_one(first_case["state"], first_case["questions"])+ cold_ms = (time.perf_counter() - t_cold_start) * 1000.0++ print(f"[bench_python] {checkpoint}: cold={cold_ms:.0f} ms, running warm-up...", flush=True)++ # ── warm latency ──────────────────────────────────────────────────────────+ # Cycle through all bench cases for variety; this also exercises both states.+ call_idx = 0++ def _next_call():+ nonlocal call_idx+ c = BENCH_CASES[call_idx % len(BENCH_CASES)]+ call_idx += 1+ return c["state"], c["questions"]++ # Warmup+ for _ in range(N_WARMUP):+ s, q = _next_call()+ agent.system_one(s, q)++ # Timed calls+ times_ms = []+ for _ in range(N_WARM):+ s, q = _next_call()+ t0 = time.perf_counter()+ agent.system_one(s, q)+ times_ms.append((time.perf_counter() - t0) * 1000.0)++ times_ms.sort()+ median_ms = times_ms[len(times_ms) // 2]+ p95_ms = times_ms[int(len(times_ms) * 0.95)]++ # ── peak memory ───────────────────────────────────────────────────────────+ peak_mb = _peak_wset_mb()++ print(+ f"[bench_python] {checkpoint}: warm median={median_ms:.1f} ms p95={p95_ms:.1f} ms"+ + (f" peak={peak_mb:.0f} MB" if peak_mb else ""),+ flush=True,+ )++ # ── answer snapshot ───────────────────────────────────────────────────────+ answers = {}+ for case in BENCH_CASES:+ result = agent.system_one(case["state"], case["questions"])+ answers[case["name"]] = result["answers"]++ import torch++ return {+ "checkpoint": checkpoint,+ "runtime": "Python SDK + PyTorch (CPU)" if runtime == "torch" else "Python + ONNX Runtime",+ "threads": f"torch default ({torch.get_num_threads()})" if runtime == "torch" else "intra=1 inter=1",+ "import_ms": round(import_ms, 1),+ "cold_ms": round(cold_ms, 1),+ "warm_times_ms": [round(t, 2) for t in times_ms],+ "warm_median_ms": round(median_ms, 1),+ "warm_p95_ms": round(p95_ms, 1),+ "n_warmup": N_WARMUP,+ "n_warm": N_WARM,+ "peak_wset_mb": peak_mb,+ "answers": answers,+ }+++def get_versions() -> dict:+ info = {}+ try:+ import onnxruntime as ort+ info["onnxruntime"] = ort.__version__+ except ImportError:+ info["onnxruntime"] = None+ try:+ import torch+ info["torch"] = torch.__version__+ except ImportError:+ info["torch"] = None+ try:+ import transformers+ info["transformers"] = transformers.__version__+ except ImportError:+ info["transformers"] = None+ try:+ import numpy as np+ info["numpy"] = np.__version__+ except ImportError:+ info["numpy"] = None+ try:+ import laya+ info["laya"] = laya.__version__+ except Exception:+ info["laya"] = None+ info["python"] = sys.version+ return info+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument(+ "--checkpoint",+ choices=CHECKPOINTS + ["all"],+ default="multilingual",+ metavar="{multilingual,english,typed-decisions,all}",+ )+ ap.add_argument(+ "--runtime",+ choices=["torch", "onnx"],+ default="torch",+ help="torch: the public SDK (laya.load, PyTorch eager). onnx: the SDK code path with "+ "the exported ONNX graph as the forward pass, single-threaded.",+ )+ ap.add_argument("--output", default=None, help="Write JSON results to this file (default: stdout only)")+ args = ap.parse_args()++ checkpoints = CHECKPOINTS if args.checkpoint == "all" else [args.checkpoint]++ results = {"versions": None, "checkpoints": {}}++ # Note: "all" shares one process, so peak memory and cold latency of later checkpoints+ # are inflated. Run one checkpoint per process for publishable numbers.+ for cp in checkpoints:+ results["checkpoints"][cp] = bench_checkpoint(cp, args.runtime)+ # Collected after the runs so the imports it triggers are not charged to import_ms.+ results["versions"] = get_versions()++ out = json.dumps(results, indent=2, ensure_ascii=False)+ print("\n" + out)+ if args.output:+ with open(args.output, "w", encoding="utf-8") as f:+ f.write(out)+ print(f"\nResults written to {args.output}", flush=True)++ return 0+++if __name__ == "__main__":+ raise SystemExit(main())