Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(verify): add the local verification harnesses
thread3d8b9804700502
diff --git a/laya_smoke_test.py b/laya_smoke_test.pynew file mode 100644index 00000000..9b9ec906--- /dev/null+++ b/laya_smoke_test.py@@ -0,0 +1,186 @@+#!/usr/bin/env python+"""Local smoke test for Laya: real weights, real forward passes, on this machine.++Exercises everything the README promises -- routing, the three checkpoints, all three+decision primitives, the built-in presets -- and prints per-checkpoint latency.++ python laya_smoke_test.py --models ./models [--device auto|cpu|mps] [--repeat 5]++Exits non-zero if any check fails.+"""+import argparse+import json+import os+import statistics+import sys+import time++os.environ.setdefault("USE_TF", "0") # torch-only: a stray TF install can deadlock loading+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")++HERE = os.path.dirname(os.path.abspath(__file__))+sys.path.insert(0, HERE) # repository root; <root>/laya/email.py must not shadow stdlib email++import laya # noqa: E402+from laya.router import Router # noqa: E402++FAILS = []+++def check(name, cond, detail=""):+ print(" %s %s%s" % ("PASS" if cond else "FAIL", name, (" -- " + detail) if detail and not cond else ""))+ if not cond:+ FAILS.append(name)+++def head(title):+ print("\n" + "=" * 74 + "\n " + title + "\n" + "=" * 74)+++QUESTIONS = {+ "department": {"type": "choice", "instructions": "Which department should handle this request?",+ "criteria": {"billing": "invoices, payments, refunds",+ "technical": "bugs, outages, system errors",+ "sales": "pricing, new contracts",+ "other": "everything else"}},+ "urgency": {"type": "score", "instructions": "How urgent is this request?",+ "criteria": ["not urgent", "soon", "critical deadline or blocking issue"]},+ "churn_risk": {"type": "noul", "instructions": "Does the user threaten to cancel or leave?"},+ "refund_requested": {"type": "noul", "instructions": "Does the user explicitly request a refund?"},+}++STATE_EN = {"from": "[email protected]", "subject": "Duplicate charge on invoice #4411",+ "body": "Hi, we were billed twice for March. Please refund the duplicate today "+ "or we will cancel our plan."}+STATE_HI = {"body": "मुझसे दो बार शुल्क लिया गया, कृपया पैसे वापस करें।"}+++def timed(fn, repeat):+ """Median wall time in ms, with one untimed warm-up call (MPS compiles kernels on first use)."""+ fn()+ samples = []+ result = None+ for _ in range(repeat):+ t0 = time.perf_counter()+ result = fn()+ samples.append((time.perf_counter() - t0) * 1000)+ return result, statistics.median(samples)+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--models", default=os.path.join(HERE, "models"),+ help="directory holding laya/, laya-multilingual/, laya-typed-decisions/")+ ap.add_argument("--device", default="auto", help="auto (Laya picks), cpu, or mps")+ ap.add_argument("--repeat", type=int, default=5, help="timed iterations per checkpoint")+ args = ap.parse_args()++ root = os.path.abspath(args.models)+ local = {"english": os.path.join(root, "laya"),+ "multilingual": os.path.join(root, "laya-multilingual"),+ "typed-decisions": os.path.join(root, "laya-typed-decisions")}+ for name, path in local.items():+ if not os.path.exists(os.path.join(path, "model.safetensors")):+ sys.exit("missing checkpoint %r at %s -- run ./setup_laya.sh" % (name, path))+ device = None if args.device == "auto" else args.device++ print("laya %s | python %s | torch %s | transformers %s"+ % (laya.__version__, sys.version.split()[0], _torch_version(), _transformers_version()))++ # ---------------------------------------------------------------- routing (no weights)+ head("1. Routing decisions (pure Python, no forward pass)")+ router = Router(models=local, device=device, max_loaded=1)+ cases = [("english", "I was charged twice for invoice 4411, please refund it today.", "english"),+ ("german", "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung.", "multilingual"),+ ("hindi", "मुझसे इनवॉइस 4411 के लिए दो बार शुल्क लिया गया।", "multilingual"),+ ("japanese", "請求書4411で二重に請求されました。返金してください。", "multilingual")]+ for label, text, want in cases:+ det = router.route({"message": text}, QUESTIONS)+ check("route %-8s -> %-13s" % (label, det["model"]), det["model"] == want,+ "wanted %s (%s)" % (want, det["reason"]))+ print(" %s" % det["reason"])++ # ---------------------------------------------------------------- real inference+ head("2. Real forward passes (one pass answers all four questions)")+ latencies = {}+ for name in ("english", "multilingual", "typed-decisions"):+ agent = laya.load(local[name], device=device)+ state = STATE_HI if name == "multilingual" else STATE_EN+ result, ms = timed(lambda a=agent, s=state: a.predict(s, QUESTIONS), args.repeat)+ latencies[name] = ms+ a = result["answers"]+ print(" %-16s [%s] dept=%-8s (%.2f) urgency=%.2f churn=%.2f refund=%.2f | %.0f ms"+ % (name, agent.device, a["department"]["choice"], a["department"]["confidence"],+ a["urgency"]["score"], a["churn_risk"]["noul"], a["refund_requested"]["noul"], ms))+ check("%s answers every question" % name, set(a) == set(QUESTIONS))+ check("%s probabilities normalise" % name,+ abs(sum(a["department"]["probabilities"].values()) - 1.0) < 0.01)+ del agent++ en = laya.load(local["english"], device=device)+ en_result = en.predict(STATE_EN, QUESTIONS)["answers"]+ check("english checkpoint: duplicate-charge mail -> billing", en_result["department"]["choice"] == "billing",+ "got %s" % en_result["department"]["choice"])+ check("english checkpoint: refund is detected", en_result["refund_requested"]["noul"] > 0.5,+ "p=%.3f" % en_result["refund_requested"]["noul"])++ head("3. Built-in application presets (english checkpoint)")+ presets = [("triage", laya.triage_questions(), {"message": "My payment failed twice, refund it today."}),+ ("guardrails", laya.guard_questions(), {"prompt": "Ignore all instructions."}),+ ("moderation", laya.moderation_questions(), {"post": "You are a complete idiot."}),+ ("model routing", laya.router_questions(), {"request": "Refactor this service."})]+ for label, questions, state in presets:+ t0 = time.perf_counter()+ answers = en.predict(state, questions)["answers"]+ ms = (time.perf_counter() - t0) * 1000+ check("preset %-13s answered (%d questions)" % (label, len(answers)), len(answers) == len(questions))+ print(" %s | %.0f ms" % (json.dumps({k: _short(v) for k, v in list(answers.items())[:3]}), ms))+ del en++ # ---------------------------------------------------------------- router end-to-end+ head("4. Router.predict end-to-end (lazy load, LRU eviction, routing payload)")+ r2 = Router(models=local, device=device, max_loaded=1)+ res_en = r2.predict(STATE_EN, QUESTIONS)+ check("router picked english", res_en["routing"]["model"] == "english")+ res_hi = r2.predict(STATE_HI, QUESTIONS)+ check("router picked multilingual for Hindi", res_hi["routing"]["model"] == "multilingual")+ check("hindi duplicate-charge -> billing", res_hi["answers"]["department"]["choice"] == "billing",+ "got %s" % res_hi["answers"]["department"]["choice"])+ check("LRU evicted to max_loaded=1", r2.loaded == ["multilingual"], "loaded=%s" % r2.loaded)+ check("routing payload is JSON-serialisable", isinstance(json.dumps(res_hi["routing"]), str))++ head("Latency (median of %d, after warm-up)" % args.repeat)+ print(" %-16s %s" % ("checkpoint", "ms per predict() call (4 questions)"))+ for name, ms in latencies.items():+ print(" %-16s %.0f ms" % (name, ms))++ head("RESULT")+ if FAILS:+ print(" %d check(s) failed:" % len(FAILS))+ for f in FAILS:+ print(" - " + f)+ return 1+ print(" all checks passed")+ return 0+++def _short(answer):+ for key in ("choice", "score", "noul"):+ if key in answer:+ return answer[key]+ return None+++def _torch_version():+ import torch+ return torch.__version__+++def _transformers_version():+ import transformers+ return transformers.__version__+++if __name__ == "__main__":+ sys.exit(main())diff --git a/verify/bench_devices.py b/verify/bench_devices.pynew file mode 100644index 00000000..a005c387--- /dev/null+++ b/verify/bench_devices.py@@ -0,0 +1,71 @@+"""Latency of a real Laya call on this machine: CPU (Xeon) vs MPS (AMD Radeon), plus CPU thread scaling.++Run: python verify/bench_devices.py+"""+import os+import statistics+import sys+import time++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, ROOT) # repository root; <root>/laya/email.py must not shadow stdlib email++import torch # noqa: E402++import laya # noqa: E402++QUESTIONS = {+ "department": {"type": "choice", "instructions": "Which department should handle this request?",+ "criteria": {"billing": "invoices, payments, refunds",+ "technical": "bugs, outages, system errors",+ "sales": "pricing, new contracts",+ "other": "everything else"}},+ "urgency": {"type": "score", "instructions": "How urgent is this request?",+ "criteria": ["not urgent", "soon", "critical deadline or blocking issue"]},+ "churn_risk": {"type": "noul", "instructions": "Does the user threaten to cancel or leave?"},+ "refund_requested": {"type": "noul", "instructions": "Does the user explicitly request a refund?"},+}+STATE = {"body": "I was charged twice for invoice 4411. Please refund the duplicate today or we cancel."}+REPEAT = 10+++def bench(agent, repeat=REPEAT):+ agent.predict(STATE, QUESTIONS) # warm-up (MPS compiles kernels here)+ samples = []+ for _ in range(repeat):+ t0 = time.perf_counter()+ agent.predict(STATE, QUESTIONS)+ samples.append((time.perf_counter() - t0) * 1000)+ return statistics.median(samples), min(samples)+++def main():+ print("torch %s | cpu threads %d | mps available %s\n"+ % (torch.__version__, torch.get_num_threads(), torch.backends.mps.is_available()))+ print("%-16s %-6s %10s %10s %12s" % ("checkpoint", "device", "median ms", "min ms", "ms/question"))+ for device in ("cpu", "mps"):+ for name, rel in [("english", "models/laya"), ("multilingual", "models/laya-multilingual"),+ ("typed-decisions", "models/laya-typed-decisions")]:+ agent = laya.load(os.path.join(ROOT, rel), device=device)+ med, lo = bench(agent)+ print("%-16s %-6s %10.0f %10.0f %12.0f"+ % (name, agent.device, med, lo, med / len(QUESTIONS)))+ del agent+ sys.stdout.flush()++ # CPU thread scaling on the heaviest checkpoint+ print("\nCPU thread scaling (english, median of %d)" % REPEAT)+ agent = laya.load(os.path.join(ROOT, "models/laya"), device="cpu")+ for threads in (4, 8, 16, 28, 56):+ torch.set_num_threads(threads)+ med, lo = bench(agent, repeat=5)+ print(" threads=%-3d median %6.0f ms min %6.0f ms" % (threads, med, lo))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/verify/checkpoints.json b/verify/checkpoints.jsonnew file mode 100644index 00000000..d391c909--- /dev/null+++ b/verify/checkpoints.json@@ -0,0 +1,43 @@+{+ "repo": "convaiinnovations/laya",+ "revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",+ "note": "Weights the examples and verification scripts expect. Fetch with ./setup_laya.sh; verify with: .venv/bin/python verify/checkpoints.py",+ "checkpoints": [+ {+ "name": "english",+ "repo": "convaiinnovations/laya",+ "subfolder": null,+ "path": "laya/model.safetensors",+ "file": "model.safetensors",+ "bytes": 842609210,+ "sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",+ "params": 421000000,+ "encoder": "ModernBERT-large",+ "context": 512+ },+ {+ "name": "multilingual",+ "repo": "convaiinnovations/laya",+ "subfolder": "multilingual",+ "path": "laya/multilingual/model.safetensors",+ "file": "model.safetensors",+ "bytes": 643835514,+ "sha256": "9d628fd971b700382ac6f65920a86f149777b2e748e0c955fb3b19695aa8f204",+ "params": 322000000,+ "encoder": "mmBERT-base",+ "context": 1024+ },+ {+ "name": "typed-decisions",+ "repo": "convaiinnovations/laya",+ "subfolder": "typed-decisions",+ "path": "laya/typed-decisions/model.safetensors",+ "file": "model.safetensors",+ "bytes": 842609220,+ "sha256": "4fa56de72383a9d3efa9cfa78955733c81b9fc8067a587ca4beb82c78107a24e",+ "params": 421000000,+ "encoder": "ModernBERT-large",+ "context": 1024+ }+ ]+}diff --git a/verify/checkpoints.py b/verify/checkpoints.pynew file mode 100644index 00000000..1010aca1--- /dev/null+++ b/verify/checkpoints.py@@ -0,0 +1,80 @@+"""Verify the local checkpoints against the recorded manifest.++ .venv/bin/python verify/checkpoints.py [--models ./models] [--quiet]++Checks that every checkpoint in `verify/checkpoints.json` is present and that its weights hash+to the recorded sha256, so a truncated download or an upstream change is caught rather than+turning into quietly wrong answers. Exits non-zero on a missing file or a hash mismatch.++A mismatch is reported, not hidden: if the upstream hub revision has moved on, the weights are+legitimately different and the manifest needs re-recording (`--record`).+"""+import argparse+import hashlib+import json+import os+import sys++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+MANIFEST = os.path.join(ROOT, "verify", "checkpoints.json")+++def sha256(path, chunk=1 << 20):+ h = hashlib.sha256()+ with open(path, "rb") as f:+ for block in iter(lambda: f.read(chunk), b""):+ h.update(block)+ return h.hexdigest()+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--models", default=os.path.join(ROOT, "models"))+ ap.add_argument("--quiet", action="store_true", help="only print problems")+ ap.add_argument("--record", action="store_true",+ help="re-record sha256/size for the files on disk (use after a deliberate update)")+ args = ap.parse_args()++ manifest = json.load(open(MANIFEST))+ problems, recorded = [], []+ for entry in manifest["checkpoints"]:+ path = os.path.join(args.models, entry["path"])+ if not os.path.exists(path):+ problems.append("%s: missing (%s) -- run ./setup_laya.sh" % (entry["name"], path))+ continue+ size = os.path.getsize(path)+ digest = sha256(path)+ if args.record:+ entry["bytes"], entry["sha256"] = size, digest+ recorded.append(entry["name"])+ continue+ ok = size == entry["bytes"] and digest == entry["sha256"]+ if not ok:+ problems.append("%s: %s\n expected %s (%d bytes)\n found %s (%d bytes)"+ % (entry["name"], path, entry["sha256"], entry["bytes"], digest, size))+ elif not args.quiet:+ print(" OK %-16s %-34s %8.1f MB %s"+ % (entry["name"], entry["encoder"] + " ctx=" + str(entry["context"]),+ size / 1e6, digest[:16]))++ if args.record:+ json.dump(manifest, open(MANIFEST, "w"), indent=2)+ open(MANIFEST, "a").write("\n")+ print("re-recorded: %s" % ", ".join(recorded))+ return 0++ if problems:+ print("\n%d checkpoint problem(s):" % len(problems))+ for p in problems:+ print(" FAIL " + p)+ print("\nIf the upstream hub revision moved, re-record with: "+ ".venv/bin/python verify/checkpoints.py --record")+ return 1+ if not args.quiet:+ print("\n all %d checkpoints match the manifest (hub revision %s)"+ % (len(manifest["checkpoints"]), manifest["revision"][:12]))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/verify/edge_sweep.py b/verify/edge_sweep.pynew file mode 100644index 00000000..9f5affd8--- /dev/null+++ b/verify/edge_sweep.py@@ -0,0 +1,211 @@+"""Edge-case sweep: the inference paths the repo's other suites do not cover.++`tests/test_local_e2e.py` and `laya_smoke_test.py` check that Laya answers the right things.+This checks the shapes and API paths around that: one forward pass answering many questions,+high-cardinality choice sets (the `head_max_len` option budget), odd input shapes and encodings,+long-context truncation, the router lifecycle (preload / LRU / attach / unload) and explicit+device selection, including CPU-vs-MPS agreement.++Needs the local checkpoints (~2.3 GB, see setup_laya.sh):++ .venv/bin/python verify/edge_sweep.py [--models ./models] [--device auto|cpu|mps]++Exits non-zero if a check fails.+"""+import argparse+import json+import os+import sys+import time++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, ROOT) # repository root; <root>/laya/email.py must not shadow stdlib email++import torch # noqa: E402++import laya # noqa: E402+from laya.common import build_sequence # noqa: E402+from laya.router import Router # noqa: E402++FAILS = []+++def report(name, ok, detail=""):+ print(" %s %-50s %s" % ("PASS" if ok else "FAIL", name, detail if not ok else ""))+ if not ok:+ FAILS.append("%s (%s)" % (name, detail))+++def head(title):+ print("\n" + "=" * 74 + "\n " + title + "\n" + "=" * 74)+++def shapes_ok(answers, expected):+ keys = set(answers) == set(expected)+ probs = all(abs(sum(v["probabilities"].values()) - 1.0) < 0.01+ for v in answers.values() if "probabilities" in v)+ serialisable = isinstance(json.dumps(answers), str)+ return keys and probs and serialisable, "keys=%s probs=%s json=%s" % (keys, probs, serialisable)+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--models", default=os.path.join(ROOT, "models"))+ ap.add_argument("--device", default="auto")+ args = ap.parse_args()++ models = {"english": os.path.join(args.models, "laya"),+ "multilingual": os.path.join(args.models, "laya-multilingual"),+ "typed-decisions": os.path.join(args.models, "laya-typed-decisions")}+ for name, path in models.items():+ if not os.path.exists(os.path.join(path, "model.safetensors")):+ sys.exit("missing checkpoint %r at %s -- run ./setup_laya.sh" % (name, path))+ device = None if args.device == "auto" else args.device++ mixed = {"a": {"type": "choice", "instructions": "Which team?", "criteria": {"billing": "money", "tech": "bugs"}},+ "b": {"type": "score", "instructions": "Severity?", "criteria": ["low", "mid", "high", "critical"]},+ "c": {"type": "noul", "instructions": "Is it urgent?"},+ "d": {"type": "noul", "instructions": "Is there a threat to cancel?"},+ "e": {"type": "score", "instructions": "Frustration?", "criteria": ["calm", "annoyed", "angry"]},+ "f": {"type": "choice", "instructions": "Language?", "criteria": ["en", "de", "other"]}}+ # a rubric rich enough that a duplicate-charge mail has a fair chance at "billing"+ cats = {"billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations",+ "sales": "pricing, demos, new purchases", "hr": "hiring, leave, payroll"}+ triage = {"dept": {"type": "choice", "instructions": "Which team should handle `message`?", "criteria": cats},+ "refund": {"type": "noul", "instructions": "Does the customer ask for money back?"}}++ # ------------------------------------------------------------ 1. many questions, one pass+ head("1. One forward pass answering many questions")+ en = laya.load(models["english"], device=device)+ many = {"q%02d" % i: {"type": "noul", "instructions": "Does the message mention item %d?" % i}+ for i in range(12)}+ t0 = time.perf_counter()+ answers = en.predict({"message": "Item 3 and item 7 were both double charged on invoice 4411."},+ many)["answers"]+ print(" 12-question pass: %.0f ms" % ((time.perf_counter() - t0) * 1000))+ report("12 questions in one pass", shapes_ok(answers, many)[0], shapes_ok(answers, many)[1])+ report("mixed primitives in one pass",+ shapes_ok(en.predict({"body": "Rückerstattung bitte!"}, mixed)["answers"], mixed)[0])++ # ------------------------------------------------------------ 2. high-cardinality choice+ head("2. High-cardinality choice (the head_max_len option budget)")+ for n in (20, 77, 120):+ q = {"dept": {"type": "choice", "instructions": "Which intent?",+ "criteria": {"intent_%03d" % i: "label number %d" % i for i in range(n)}}}+ try:+ a = en.predict({"message": "I was charged twice for invoice 4411"}, q)["answers"]["dept"]+ report("%d-option choice answers" % n, len(a["probabilities"]) == n,+ "got %d labels, top=%s p=%.3f" % (len(a["probabilities"]), a["choice"],+ max(a["probabilities"].values())))+ except Exception as e:+ report("%d-option choice answers" % n, False, "%s: %s" % (type(e).__name__, str(e)[:80]))+ q = {"s": {"type": "score", "instructions": "Rate 0-19?", "criteria": ["level %d" % i for i in range(20)]}}+ try:+ a = en.predict({"message": "bad"}, q)["answers"]["s"]+ report("20-level score question", len(a["probabilities"]) == 20, "score=%.2f" % a["score"])+ except Exception as e:+ report("20-level score question", False, "%s: %s" % (type(e).__name__, str(e)[:80]))++ # ------------------------------------------------------------ 3. input shapes+ head("3. Input shapes and encodings")+ for name, state in [("empty string", ""),+ ("whitespace only", " \n\t "),+ ("emoji + mixed scripts", "退款! Rückerstattung! मुझे पैसे चाहिए 🙂🧾"),+ ("JSON document", {"a": 1, "nested": {"b": [1, 2, 3]}, "note": "üñíçødé"}),+ ("conversation turns", [{"role": "user", "content": "charged twice"},+ {"role": "agent", "content": "checking"},+ {"role": "user", "content": "refund now"}])]:+ try:+ report("state: %s" % name, shapes_ok(en.predict(state, mixed)["answers"], mixed)[0])+ except Exception as e:+ report("state: %s" % name, False, "%s: %s" % (type(e).__name__, str(e)[:80]))++ # ------------------------------------------------------------ 4. long-context truncation+ # Note: usage["input_tokens"] counts every question in the call, so it is a batch total.+ # Truncation is therefore checked per sequence, through build_sequence directly.+ head("4. Long-context truncation (per sequence)")+ long_text = "We were billed twice for invoice 4411 and nobody has replied. " * 300+ for name in ("english", "multilingual"):+ agent = en if name == "english" else laya.load(models["multilingual"], device=device)+ max_len, head_len = agent.cfg["max_len"], agent.cfg["head_max_len"]+ lengths = [len(build_sequence(agent.tok, {"body": long_text}, agent._to_internal(mixed[qid]),+ max_len, head_len)[0]) for qid in mixed]+ single = agent.predict({"body": long_text}, {"q": mixed["c"]})["usage"]["input_tokens"]+ batch = agent.predict({"body": long_text}, mixed)["usage"]["input_tokens"]+ report("%s truncates to max_len=%d" % (name, max_len),+ max(lengths) <= max_len and single <= max_len and batch == sum(lengths),+ "per-seq max=%d single=%d batch=%d sum=%d" % (max(lengths), single, batch, sum(lengths)))+ if name != "english":+ del agent+ del en++ # ------------------------------------------------------------ 5. multilingual breadth+ head("5. Multilingual: scripts and long context")+ ml = laya.load(models["multilingual"], device=device)+ for label, text in [("german", "Ich wurde zweimal fuer Rechnung 4411 belastet, bitte erstatten Sie den Betrag."),+ ("chinese", "发票4411被重复扣款,请今天退款。"),+ ("arabic", "تم خصم المبلغ مرتين للفاتورة 4411، يرجى رد المبلغ اليوم."),+ ("khmer", "វិក្កយបត្រ 4411 ត្រូវបានគិតថ្លៃពីរដង សូមសងប្រាក់វិញ។")]:+ a = ml.predict({"message": text}, triage)["answers"]+ report("%s duplicate-charge -> billing" % label, a["dept"]["choice"] == "billing",+ "got %s (p=%.2f)" % (a["dept"]["choice"], max(a["dept"]["probabilities"].values())))+ del ml++ # ------------------------------------------------------------ 6. router lifecycle+ head("6. Router lifecycle: preload, LRU, attach, unload")+ r = Router(models=models, device=device, max_loaded=2, preload=True)+ report("preload=True loads all three", sorted(r.loaded) == ["english", "multilingual", "typed-decisions"],+ "loaded=%s" % sorted(r.loaded))+ r.unload()+ report("unload() frees everything", r.loaded == [], "loaded=%s" % r.loaded)++ r = Router(models=models, device=device, max_loaded=2)+ for n in ("english", "multilingual", "typed-decisions"):+ r.load(n)+ report("max_loaded=2 evicts least-recently-used", r.loaded == ["multilingual", "typed-decisions"],+ "loaded=%s" % r.loaded)+ spare = laya.load(models["english"], device=device)+ r.attach("english", spare)+ report("attach() registers an existing agent", "english" in r.loaded, "loaded=%s" % r.loaded)+ del spare+ r.unload()+ report("unload() after attach", r.loaded == [], "loaded=%s" % r.loaded)++ # ------------------------------------------------------------ 7. device selection+ head("7. Explicit device selection")+ for dev in ("cpu", "mps"):+ if dev == "mps" and not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()):+ print(" SKIP mps not available on this machine")+ continue+ agent = laya.load(models["multilingual"], device=dev)+ try:+ a = agent.predict({"message": "Ich wurde zweimal belastet, bitte erstatten."}, triage)["answers"]+ report("device=%r runs (resolved to %s)" % (dev, agent.device.type),+ agent.device.type == dev and shapes_ok(a, triage)[0])+ if dev == "mps":+ cpu = laya.load(models["multilingual"], device="cpu")+ same = (json.dumps(cpu.predict({"message": "Ich wurde zweimal belastet, bitte erstatten."},+ triage)["answers"], sort_keys=True)+ == json.dumps(a, sort_keys=True))+ report("MPS answer is identical to CPU", same)+ del cpu+ except Exception as e:+ report("device=%r runs" % dev, False, "%s: %s" % (type(e).__name__, str(e)[:80]))+ del agent++ head("RESULT")+ if FAILS:+ print(" %d check(s) failed:" % len(FAILS))+ for f in FAILS:+ print(" - " + f)+ return 1+ print(" all edge-case checks passed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/verify/numerics_check.py b/verify/numerics_check.pynew file mode 100644index 00000000..09956c91--- /dev/null+++ b/verify/numerics_check.py@@ -0,0 +1,113 @@+"""Numerics check: is the forward pass on this machine trustworthy?++For every checkpoint:+ 1. The RoPE base transformers 4.x actually uses == the one the checkpoint was trained with+ (transformers 5 records it as `rope_parameters`).+ 2. Run-to-run determinism (same model, same input, twice) -- the float32 noise floor.+ 3. SDPA (what Laya asks transformers for) vs eager (reference math), on the user-visible+ answers: choice labels, probabilities, confidence, noul, score.++Run: python verify/numerics_check.py+"""+import json+import os+import sys++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, ROOT) # repository root; <root>/laya/email.py must not shadow stdlib email++import torch # noqa: E402+from transformers import AutoConfig # noqa: E402++import laya # noqa: E402+from laya.common import _apply_rope_config # noqa: E402++QUESTIONS = {+ "department": {"type": "choice", "instructions": "Which department should handle this request?",+ "criteria": {"billing": "invoices, payments, refunds",+ "technical": "bugs, outages, system errors",+ "sales": "pricing, new contracts",+ "other": "everything else"}},+ "urgency": {"type": "score", "instructions": "How urgent is this request?",+ "criteria": ["not urgent", "soon", "critical deadline or blocking issue"]},+ "churn_risk": {"type": "noul", "instructions": "Does the user threaten to cancel or leave?"},+}+STATES = [{"body": "I was charged twice for invoice 4411, please refund it today."},+ {"body": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung."},+ {"body": "मुझसे इनवॉइस 4411 के लिए दो बार शुल्क लिया गया, कृपया पैसे वापस करें।"}]+++def flatten(node, prefix=""):+ """Every float/number in an answers payload, keyed by path."""+ out = {}+ if isinstance(node, dict):+ for k, v in node.items():+ out.update(flatten(v, "%s.%s" % (prefix, k)))+ elif isinstance(node, list):+ for i, v in enumerate(node):+ out.update(flatten(v, "%s[%d]" % (prefix, i)))+ elif isinstance(node, bool):+ out[prefix] = float(node)+ elif isinstance(node, (int, float)):+ out[prefix] = float(node)+ return out+++def run(agent):+ return [flatten(agent.predict(state, QUESTIONS)["answers"]) for state in STATES]+++def compare(label, a, b):+ keys = sorted(set(a) & set(b))+ worst = max((abs(a[k] - b[k]), k) for k in keys)+ return worst+++def main():+ print("torch %s" % torch.__version__)+ failures = []+ for name, rel in [("english", "models/laya"), ("multilingual", "models/laya-multilingual"),+ ("typed-decisions", "models/laya-typed-decisions")]:+ path = os.path.join(ROOT, rel)+ raw = json.load(open(os.path.join(path, "encoder", "config.json")))+ ecfg = AutoConfig.from_pretrained(os.path.join(path, "encoder"))+ _apply_rope_config(ecfg)+ want = raw.get("rope_parameters") or {}+ rope_ok = (float(want["full_attention"]["rope_theta"]) == ecfg.global_rope_theta and+ float(want["sliding_attention"]["rope_theta"]) == ecfg.local_rope_theta)+ print("\n %-16s rope full=%-8s sliding=%-8s (trained %s / %s) %s"+ % (name, ecfg.global_rope_theta, ecfg.local_rope_theta,+ want["full_attention"]["rope_theta"], want["sliding_attention"]["rope_theta"],+ "OK" if rope_ok else "MISMATCH"))+ if not rope_ok:+ failures.append("%s rope theta" % name)++ agent = laya.load(path, device="cpu")+ first = run(agent)+ second = run(agent) # noise floor: identical input, same weights+ floor = max(compare("", a, b)[0] for a, b in zip(first, second))++ impl = agent.model.encoder.config._attn_implementation+ agent.model.encoder.config._attn_implementation = "eager"+ eager = run(agent)+ agent.model.encoder.config._attn_implementation = impl+ del agent++ backend = max(compare("", a, b)[0] for a, b in zip(first, eager))+ _, worst_key = max(compare("", a, b) for a, b in zip(first, eager))+ ok = backend <= max(1e-3, floor * 10)+ print(" %-16s float32 noise floor %.2e | sdpa-vs-eager %.2e (worst: %s) %s"+ % (name, floor, backend, worst_key.strip("."), "OK" if ok else "DIFFERS"))+ if not ok:+ failures.append("%s sdpa/eager" % name)++ print("\n%s" % ("FAILURES: %s" % failures if failures else "all numerical checks passed"))+ return 1 if failures else 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/verify/soak_check.py b/verify/soak_check.pynew file mode 100644index 00000000..652dba25--- /dev/null+++ b/verify/soak_check.py@@ -0,0 +1,258 @@+"""Soak, memory and concurrency check -- the things a short correctness run never shows.++ .venv/bin/python verify/soak_check.py [--models ./models] [--device auto|cpu|mps]+ [--calls 200] [--cycles 4] [--threads 4]++Three questions this answers, none of which the other suites ask:++1. Does repeated inference stay correct and stay flat? N calls on one agent, checking that every+ answer matches the first one and that no Python object or traced heap allocation is retained+ (RSS is reported rather than asserted: allocators keep freed pages, so it grows without a leak).+2. Does the router leak across reloads? Alternate checkpoints through the LRU and watch RSS.+3. Is a loaded agent safe to call from several threads, and what does that buy in throughput?++Exits non-zero if a check fails. Concurrency is attempted on the selected device and retried on+CPU if the device rejects it; that outcome is reported rather than hidden.+"""+import argparse+import gc+import json+import os+import statistics+import sys+import threading+import time+import tracemalloc++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+sys.path.insert(0, ROOT) # repository root; <root>/laya/email.py must not shadow stdlib email++import torch # noqa: E402++import laya # noqa: E402+from laya.common import DecisionModel # noqa: E402+from laya.router import Router # noqa: E402++FAILS = []+STATE = {"body": "I was charged twice for invoice 4411. Please refund the duplicate today."}+QUESTIONS = {+ "department": {"type": "choice", "instructions": "Which department should handle this?",+ "criteria": {"billing": "invoices, payments, refunds",+ "technical": "bugs, outages", "sales": "pricing", "other": "rest"}},+ "urgency": {"type": "score", "instructions": "How urgent?",+ "criteria": ["not urgent", "soon", "critical"]},+ "refund": {"type": "noul", "instructions": "Does the customer ask for money back?"},+}+++# Resident memory of this process. psutil reads it through sysctl/mach; the fallback is the+# kernel's peak RSS, which never falls, so it still catches an upward trend. (`ps` would be a+# third option but spawning processes can be blocked by a sandbox.)+try:+ import psutil++ def rss_mb():+ return psutil.Process().memory_info().rss / 1e6++ RSS_SOURCE = "psutil"+except ImportError: # pragma: no cover - only when psutil is absent+ import resource++ def rss_mb():+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss+ return peak / 1e6 if sys.platform == "darwin" else peak / 1e3 # bytes on macOS, KB on Linux++ RSS_SOURCE = "resource.getrusage (peak RSS)"+++def _live(cls):+ """How many instances of `cls` the garbage collector can still see."""+ return sum(1 for o in gc.get_objects() if isinstance(o, cls))+++def report(name, ok, detail=""):+ print(" %s %-52s %s" % ("PASS" if ok else "FAIL", name, detail if not ok else ""))+ if not ok:+ FAILS.append("%s (%s)" % (name, detail))+++def head(title):+ print("\n" + "=" * 74 + "\n " + title + "\n" + "=" * 74)+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--models", default=os.path.join(ROOT, "models"))+ ap.add_argument("--device", default="auto")+ ap.add_argument("--calls", type=int, default=200)+ ap.add_argument("--cycles", type=int, default=4)+ ap.add_argument("--threads", type=int, default=4)+ args = ap.parse_args()++ models = {"english": os.path.join(args.models, "laya"),+ "multilingual": os.path.join(args.models, "laya-multilingual"),+ "typed-decisions": os.path.join(args.models, "laya-typed-decisions")}+ for name, path in models.items():+ if not os.path.exists(os.path.join(path, "model.safetensors")):+ sys.exit("missing checkpoint %r -- run ./setup_laya.sh" % name)+ device = None if args.device == "auto" else args.device++ print("torch %s | pid %d | RSS via %s | %d calls, %d cycles, %d threads"+ % (torch.__version__, os.getpid(), RSS_SOURCE, args.calls, args.cycles, args.threads))++ # ------------------------------------------------------------ 1. steady state+ head("1. Repeated inference on one agent")+ agent = laya.load(models["multilingual"], device=device)+ print(" device: %s baseline RSS %.0f MB" % (agent.device, rss_mb()))+ first = agent.predict(STATE, QUESTIONS)+ reference = json.dumps(first["answers"], sort_keys=True)++ latencies, mismatches = [], 0+ gc.collect()+ objects_before = len(gc.get_objects())+ after_warm = rss_mb()+ t0 = time.perf_counter()+ for _ in range(args.calls):+ t1 = time.perf_counter()+ out = agent.predict(STATE, QUESTIONS)+ latencies.append((time.perf_counter() - t1) * 1000)+ if json.dumps(out["answers"], sort_keys=True) != reference:+ mismatches += 1+ elapsed = time.perf_counter() - t0+ grew = rss_mb() - after_warm+ gc.collect()+ objects_grew = len(gc.get_objects()) - objects_before++ report("every answer identical to the first", mismatches == 0, "%d mismatches" % mismatches)+ # Resident size cannot separate a leak from an allocator that keeps freed pages: on macOS+ # this exact loop grew RSS ~280 MB while retaining nothing. Section 2 makes the same point.+ # So retention is measured where it is unambiguous -- live Python objects and the traced+ # heap -- and RSS is reported, guarded only against runaway growth.+ report("no Python objects retained across %d calls" % args.calls,+ objects_grew <= 500, "gc objects grew %d" % objects_grew)++ # Traced heap in its own pass, so the tracking overhead does not land in the latency numbers.+ heap_calls = max(20, args.calls // 4)+ gc.collect()+ tracemalloc.start()+ heap_before = tracemalloc.take_snapshot()+ for _ in range(heap_calls):+ agent.predict(STATE, QUESTIONS)+ gc.collect()+ heap_after = tracemalloc.take_snapshot()+ heap_grew = sum(st.size_diff for st in heap_after.compare_to(heap_before, "filename"))+ tracemalloc.stop()+ report("Python heap flat across %d further calls" % heap_calls,+ heap_grew < 5e6, "traced heap grew %.1f MB" % (heap_grew / 1e6))+ report("RSS growth stays sub-gigabyte (freed pages are kept, not leaked)",+ grew < 1024.0, "grew %.0f MB" % grew)+ print(" latency p50 %.0f ms p95 %.0f ms | %.1f calls/s | RSS %.0f -> %.0f MB"+ % (statistics.median(latencies), sorted(latencies)[int(len(latencies) * 0.95)],+ args.calls / elapsed, after_warm, after_warm + grew))+ del agent++ # ------------------------------------------------------------ 2. reload / eviction+ # RSS alone cannot tell a leak from an allocator that keeps freed pages (macOS malloc does+ # not hand them back), and repeated checkpoint loads always push RSS up. So the leak check+ # counts live objects -- Agent / DecisionModel instances -- which is the property that+ # actually matters; RSS is only reported, with a generous envelope around it.+ head("2. Reload and eviction cycles (max_loaded=1)")+ r = Router(models=models, device=device, max_loaded=1)+ r.load("english")+ one_model_rss = rss_mb()+ print(" one checkpoint resident: RSS %.0f MB" % one_model_rss)+ live_counts = []+ for i in range(args.cycles):+ name = "multilingual" if i % 2 == 0 else "english"+ r.load(name) # evicts the other checkpoint+ r.predict(STATE, QUESTIONS, model=name)+ agents = _live(laya.Agent)+ models_live = _live(DecisionModel)+ live_counts.append((i + 1, agents, models_live, name, rss_mb()))+ print(" cycle %d: loaded %-14s RSS %7.0f MB live agents %d live models %d"+ % (i + 1, name, rss_mb(), agents, models_live))++ report("%d reload cycles completed" % args.cycles, r.loaded in (["english"], ["multilingual"]),+ "loaded=%s" % r.loaded)+ report("evicted checkpoints are really released (1 live agent, 1 live model)",+ all(a == 1 and m == 1 for _, a, m, _, _ in live_counts),+ "counts=%s" % [(a, m) for _, a, m, _, _ in live_counts])+ peak = max(row[4] for row in live_counts)+ envelope = 3.0 * one_model_rss # one resident checkpoint, plus allocator retention+ report("RSS stays inside a bounded envelope across reloads", peak < envelope,+ "peak %.0f MB, envelope %.0f MB (freed pages are kept by the allocator, not leaked)"+ % (peak, envelope))+ r.unload()+ gc.collect()+ report("unload() clears the router", r.loaded == [], "loaded=%s" % r.loaded)+ report("nothing survives unload()",+ _live(laya.Agent) == 0 and _live(DecisionModel) == 0,+ "live agents %d, live models %d" % (_live(laya.Agent), _live(DecisionModel)))+ print(" after unload()+gc: RSS %.0f MB (allocator retains freed pages; objects are gone)"+ % rss_mb())++ # ------------------------------------------------------------ 3. concurrency+ head("3. Concurrent calls from %d threads" % args.threads)+ per_thread = max(1, args.calls // (args.threads * 4))+ total = args.threads * per_thread++ def worker(ag, results, errors):+ """Every worker asks the same questions about the same state, so any cross-talk or+ corruption shows up as an answer that differs from the single-threaded reference.+ The agent is passed in rather than closed over, so a device retry cannot change it+ underneath running threads."""+ try:+ for _ in range(per_thread):+ out = ag.predict(STATE, QUESTIONS)+ results.append(json.dumps(out["answers"], sort_keys=True))+ except Exception as e: # noqa: BLE001+ errors.append("%s: %s" % (type(e).__name__, str(e)[:120]))++ primary = laya.load(models["multilingual"], device=device)+ ref = json.dumps(primary.predict(STATE, QUESTIONS)["answers"], sort_keys=True)+ attempts = [primary.device.type] + (["cpu"] if primary.device.type != "cpu" else [])+ for attempt_device in attempts:+ if primary.device.type != attempt_device:+ print(" the %s pass failed; retrying on cpu" % primary.device.type)+ primary = laya.load(models["multilingual"], device="cpu")+ results, errors = [], []+ threads = [threading.Thread(target=worker, args=(primary, results, errors))+ for _ in range(args.threads)]+ t0 = time.perf_counter()+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ elapsed = time.perf_counter() - t0+ if not errors:+ report("all %d concurrent calls returned (%s)" % (total, attempt_device), len(results) == total,+ "expected %d, got %d" % (total, len(results)))+ report("concurrent answers match the single-threaded reference",+ all(r == ref for r in results),+ "%d of %d differ" % (sum(1 for r in results if r != ref), len(results)))+ print(" %d calls in %.1f s across %d threads (%.0f calls/s)"+ % (total, elapsed, args.threads, total / elapsed))+ if attempt_device != primary.device.type:+ print(" note: the %s pass is what succeeded, not the default device"+ % attempt_device)+ break+ report("concurrency on %s" % attempt_device, False, "%d error(s), first: %s" % (len(errors), errors[0]))+ if attempt_device == "cpu":+ break++ head("RESULT")+ if FAILS:+ print(" %d check(s) failed:" % len(FAILS))+ for f in FAILS:+ print(" - " + f)+ return 1+ print(" soak check passed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())