Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix: harden inference and runtime parity (#319)
Kunj SHAHd0aeae31256d
diff --git a/examples/server.py b/examples/server.pyindex 32324fcc..5214a21f 100644--- a/examples/server.py+++ b/examples/server.py@@ -37,6 +37,7 @@ from typing import Any, Dict, List, Optional, Unionfrom fastapi import FastAPI, HTTPException, Requestfrom fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse+from starlette.concurrency import run_in_threadpoolfrom pydantic import BaseModel, Field, field_validator, model_validatorfrom pydantic import ValidationError as PydanticValidationError@@ -1054,8 +1055,9 @@ async def gui_predict(request: Request) -> HTMLResponse:questions = _questions(req.questions)try:started = time.perf_counter()- res = _predict(- req.state, questions, model=req.model, task=req.task, lang=req.lang+ res = await run_in_threadpool(+ _predict, req.state, questions,+ model=req.model, task=req.task, lang=req.lang,)res["_elapsed"] = time.perf_counter() - startedexcept Exception as exc:diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex 49a65cd8..11a33db7 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -145,10 +145,37 @@ export function checkQuestion(qid: string, qdef: unknown): void {throw new Error(`question ${qidStr(qid)}: a noul question takes 'criteria' as a dict with optional 'true'/'false' descriptions, or omits it`,);+ } else if (crit && typeof crit === "object" && !Array.isArray(crit)) {+ const invalid = Object.keys(crit as Record<string, unknown>).filter((key) => key !== "true" && key !== "false");+ if (invalid.length > 0) {+ throw new Error(+ `question ${qidStr(qid)}: noul criteria may contain only 'true' and 'false'; got ${JSON.stringify(invalid)}`,+ );+ }+ }+ if ("labels" in q && t !== "noul") {+ throw new Error(`question ${qidStr(qid)}: 'labels' is only supported for noul questions`);+ }+ if (t === "noul" && "labels" in q) {+ const labels = q["labels"];+ if (typeof labels !== "object" || labels === null || Array.isArray(labels)) {+ throw new Error(`question ${qidStr(qid)}: noul labels must be an object with 'false' and 'true'`);+ }+ const entries = Object.entries(labels as Record<string, unknown>);+ const keys = entries.map(([key]) => key).sort();+ if (keys.length !== 2 || keys[0] !== "false" || keys[1] !== "true" ||+ entries.some(([, value]) => typeof value !== "string" || value.trim() === "")) {+ throw new Error(`question ${qidStr(qid)}: noul labels must map exactly 'false' and 'true' to distinct non-empty strings`);+ }+ const falseLabel = String((labels as Record<string, unknown>)["false"]).trim();+ const trueLabel = String((labels as Record<string, unknown>)["true"]).trim();+ if (falseLabel === trueLabel) {+ throw new Error(`question ${qidStr(qid)}: noul labels must be distinct`);+ }}}-export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul"; ins: string; crit: unknown } {+export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } {const t = qdef["type"] as "choice" | "score" | "noul";let crit: unknown = qdef["criteria"];if (t === "choice" && Array.isArray(crit)) {@@ -157,8 +184,15 @@ export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul";crit = Object.fromEntries(Object.entries(crit as Record<string, unknown>).map(([k, v]) => [String(k).toLowerCase(), v]));}let ins: unknown = qdef["instructions"];- if (typeof ins !== "string") ins = JSON.stringify(ins);- return { t, ins: ins as string, crit };+ if (typeof ins !== "string") ins = serializeState(ins);+ const out: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } = {+ t, ins: ins as string, crit,+ };+ if (t === "noul" && qdef["labels"] && typeof qdef["labels"] === "object" && !Array.isArray(qdef["labels"])) {+ const labels = qdef["labels"] as Record<string, unknown>;+ out.labels = { false: String(labels["false"]).trim(), true: String(labels["true"]).trim() };+ }+ return out;}export function defaultTokenizer(): TokenizerLike {@@ -190,27 +224,6 @@ function tokenizerFromHF(tokenizerJson: unknown): TokenizerLike | null {};}-/** Fallback when tokenizer.json is absent: ids from rl_agent_config, if present. */-function tokenizerFromConfig(cfg: AgentCfg): TokenizerLike | null {- const c = cfg as Record<string, unknown>;- const num = (v: unknown): number | null =>- typeof v === "number" && Number.isFinite(v) ? v : null;- const cls = num(c["cls_token_id"]) ?? num(c["cls_id"]);- const sep = num(c["sep_token_id"]) ?? num(c["sep_id"]);- const mask = num(c["mask_token_id"]) ?? num(c["mask_id"]);- const pad = num(c["pad_token_id"]) ?? num(c["pad_id"]);- if (cls === null && sep === null && mask === null && pad === null) return null;- const d = defaultTokenizer();- return {- clsId: cls ?? d.clsId,- sepId: sep ?? d.sepId,- maskId: mask ?? d.maskId,- padId: pad ?? d.padId,- maskToken: typeof c["mask_token"] === "string" ? (c["mask_token"] as string) : d.maskToken,- encode: d.encode,- };-}-const r4 = (v: number): number => Math.round(v * 1e4) / 1e4;export class Agent extends HookRegistry {@@ -341,7 +354,7 @@ export class Agent extends HookRegistry {return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } };}const items: { ids: number[]; markers: number[]; qtype: number }[] = [];- const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[] = [];+ const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }[] = [];// The state text is shared by every question and is usually the longest text in the// sequence — encode it once and compose the per-question prefixes onto it.const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));@@ -350,7 +363,9 @@ export class Agent extends HookRegistry {const q = toInternal(questions[qid]);internals.push(q);const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen);- const { ids: seq, markers } = sequenceWithState(prefix, stAll, this.tok.sepId, this.maxLen);+ const { ids: seq, markers } = sequenceWithState(+ prefix, stAll, this.tok.sepId, this.maxLen, Array.isArray(state),+ );if (markers.length !== renderOptions(q).length) {throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`);}@@ -362,6 +377,16 @@ export class Agent extends HookRegistry {const nTokens = batch.attentionMask.flat().reduce((a, b) => a + b, 0);const { lastHidden } = await this.provider.runEncoder(batch);const { logits, act } = await this.provider.runHead(lastHidden, batch);+ if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < ids.length || act.length < ids.length) {+ throw new Error("model provider returned fewer output rows than input items");+ }+ for (let r = 0; r < ids.length; r++) {+ const k = items[r].markers.length;+ if (!Array.isArray(logits[r]) || logits[r].length < k || !Array.isArray(act[r]) || act[r].length === 0 ||+ logits[r].some((v) => !Number.isFinite(v)) || act[r].some((v) => !Number.isFinite(v))) {+ throw new Error(`model provider returned invalid output for question ${ids[r]}`);+ }+ }const answers: Record<string, SystemAnswer> = {};for (let r = 0; r < ids.length; r++) {@@ -454,12 +479,21 @@ export class Agent extends HookRegistry {dir = bundle.dir;provider = await createNodeProvider(dir, { device: opts?.device, numThreads: opts?.numThreads });}- let tok: TokenizerLike | null = null;+ if (!tokenizerJson) {+ throw new Error(+ `Incompatible model: tokenizer.json is missing or invalid in ${JSON.stringify(dir)}`,+ );+ }+ let tok: TokenizerLike;try {- tok = tokenizerJson ? tokenizerFromHF(tokenizerJson) : tokenizerFromConfig(cfg);- } catch {- tok = null;+ const parsed = tokenizerFromHF(tokenizerJson);+ if (!parsed) throw new Error("unsupported tokenizer.json format");+ tok = parsed;+ } catch (error) {+ throw new Error(+ `Incompatible model: tokenizer.json is missing or invalid in ${JSON.stringify(dir)}: ${String(error)}`,+ );}- return new Agent({ provider, tok: tok ?? defaultTokenizer(), cfg });+ return new Agent({ provider, tok, cfg });}}diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex f0b176a7..f46f1b51 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -1,11 +1,17 @@import type { TokenizerLike } from "./tokenizer.js";export type QType = "choice" | "score" | "noul";-export interface InternalQ { t: QType; ins: string; crit: unknown }+export interface InternalQ { t: QType; ins: string; crit: unknown; labels?: { false: string; true: string } }/** Python `json.dumps(v, ensure_ascii=False)` replica: separators (", ", ": "),unicode raw, unknown types fall back to undefined (caller applies str()). */function pyJson(v: unknown): string | undefined {if (v === null) return "null";- if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") return JSON.stringify(v);+ if (typeof v === "string" || typeof v === "boolean") return JSON.stringify(v);+ if (typeof v === "number") {+ if (Number.isNaN(v)) return "NaN";+ if (v === Infinity) return "Infinity";+ if (v === -Infinity) return "-Infinity";+ return JSON.stringify(v);+ }if (Array.isArray(v)) return `[${v.map((x) => pyJson(x) ?? "null").join(", ")}]`;if (typeof v === "object") {const proto = Object.getPrototypeOf(v);@@ -36,10 +42,11 @@ export function renderOptions(q: InternalQ): string[] {return (q.crit as unknown[]).map((c, i) => `level ${i}: ${renderCriterion(c)}`);}const crit = (q.crit ?? {}) as Record<string, unknown>;+ const labels = q.labels ?? { false: "false", true: "true" };const f = crit["false"], t = crit["true"];return [- "false: " + (f !== null && f !== undefined && f !== "" ? renderCriterion(f) : "no, the statement does not hold"),- "true: " + (t !== null && t !== undefined && t !== "" ? renderCriterion(t) : "yes, the statement holds"),+ labels.false + ": " + (f !== null && f !== undefined && f !== "" ? renderCriterion(f) : "no, the statement does not hold"),+ labels.true + ": " + (t !== null && t !== undefined && t !== "" ? renderCriterion(t) : "yes, the statement holds"),];}export interface QuestionPrefix {@@ -102,6 +109,7 @@ export function confidenceFromProbs(p: number[]): number {}export const TEMP_MIN = 0.5, TEMP_MAX = 5.0;export function clampTemperature(t: unknown): number {+ if (t === null || t === undefined || t === "" || typeof t === "boolean") return 1.0;const f = typeof t === "number" ? t : Number(t);if (!Number.isFinite(f)) return 1.0;return Math.min(TEMP_MAX, Math.max(TEMP_MIN, f));diff --git a/laya-ts/src/providers.ts b/laya-ts/src/providers.tsindex 91084b08..01670908 100644--- a/laya-ts/src/providers.ts+++ b/laya-ts/src/providers.ts@@ -195,7 +195,7 @@ export async function loadNodeBundle(await fs.mkdir(cache, { recursive: true });const token =opts?.token ?? (typeof process !== "undefined" ? (process as any).env?.["HF_TOKEN"] : undefined);- for (const f of ["rl_agent_config.json", "tokenizer.json", "encoder.onnx", "head.onnx"]) {+ for (const f of ["rl_agent_config.json", "tokenizer.json", "tokenizer/tokenizer.json", "encoder.onnx", "head.onnx"]) {try {await fs.stat(path.join(cache, f));} catch {@@ -209,7 +209,9 @@ export async function loadNodeBundle(}continue;}- await fs.writeFile(path.join(cache, f), new Uint8Array(await res.arrayBuffer()));+ const target = path.join(cache, f);+ await fs.mkdir(path.dirname(target), { recursive: true });+ await fs.writeFile(target, new Uint8Array(await res.arrayBuffer()));}}dir = cache;@@ -223,10 +225,13 @@ export async function loadNodeBundle();}let tokenizerJson: unknown | null = null;- try {- tokenizerJson = JSON.parse(await fs.readFile(path.join(dir, "tokenizer.json"), "utf8"));- } catch {- tokenizerJson = null;+ for (const candidate of ["tokenizer.json", "tokenizer/tokenizer.json"]) {+ try {+ tokenizerJson = JSON.parse(await fs.readFile(path.join(dir, candidate), "utf8"));+ break;+ } catch {+ // Try the next supported Hugging Face layout.+ }}return { dir, cfg, tokenizerJson };}@@ -257,10 +262,13 @@ export async function loadWebBundle(throw new Error(`Incompatible model: ${JSON.stringify(repoOrUrl)} does not contain 'rl_agent_config.json'.`);}let tokenizerJson: unknown | null = null;- try {- tokenizerJson = await fetchJson(`${base}/tokenizer.json`);- } catch {- tokenizerJson = null;+ for (const candidate of ["tokenizer.json", "tokenizer/tokenizer.json"]) {+ try {+ tokenizerJson = await fetchJson(`${base}/${candidate}`);+ break;+ } catch {+ // Try the next supported Hugging Face layout.+ }}return { dir: base, cfg, tokenizerJson };}diff --git a/laya-ts/tests/audit-regressions.test.ts b/laya-ts/tests/audit-regressions.test.tsnew file mode 100644index 00000000..fb3120c6--- /dev/null+++ b/laya-ts/tests/audit-regressions.test.ts@@ -0,0 +1,76 @@+import { describe, expect, it } from "vitest";+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { Agent, checkQuestion, toInternal } from "../src/agent.js";+import { buildSequence } from "../src/common.js";+import { loadNodeBundle } from "../src/providers.js";++const tokenizer = () => ({+ clsId: 1,+ sepId: 2,+ maskId: 3,+ padId: 0,+ maskToken: "[MASK]",+ encode(text: string): number[] {+ return Array.from(text).map((c) => c.codePointAt(0) ?? 0);+ },+});++const provider = (calls: any[] = []) => ({+ async runEncoder(batch: any) {+ calls.push(batch);+ return { lastHidden: [[[0, 0]]] };+ },+ async runHead(_hidden: any, _batch: any) {+ return { logits: [[1, 0]], act: [[1, 0]] };+ },+});++describe("audit regressions", () => {+ it("preserves the newest conversation turn", () => {+ const q = toInternal({ type: "noul", instructions: "?" });+ const result = buildSequence(tokenizer(), [+ { text: "old context that must be truncated" },+ { text: "newest refund request" },+ ], q, 64, 32, undefined, true);+ const rendered = result.ids.map((id) => String.fromCodePoint(id)).join("");+ expect(rendered).toContain("refund request");+ });++ it("renders custom noul labels and validates unsupported labels", () => {+ const q = toInternal({+ type: "noul",+ instructions: "?",+ criteria: {},+ labels: { false: "No", true: "Yes" },+ });+ const sequence = buildSequence(tokenizer(), "state", q, 64, 32);+ const rendered = sequence.ids.map((id) => String.fromCodePoint(id)).join("");+ expect(rendered).toContain("No");+ expect(rendered).toContain("Yes");+ expect(() => checkQuestion("q", { type: "noul", instructions: "?", labels: { false: "No" } })).toThrow("labels");+ expect(() => checkQuestion("q", { type: "choice", instructions: "?", criteria: ["yes"], labels: { false: "No", true: "Yes" } })).toThrow("labels");+ });++ it("rejects malformed provider output", async () => {+ const agent = new Agent({ provider: {+ async runEncoder(_batch: any) { return { lastHidden: [[[0, 0]]] }; },+ async runHead(_hidden: any, _batch: any) { return { logits: [[]], act: [[1, 0]] }; },+ }, tok: tokenizer() } as any);+ await expect(agent.systemOne("state", { q: { type: "noul", instructions: "?" } })).rejects.toThrow("invalid output");+ });++ it("loads the nested Hugging Face tokenizer layout", async () => {+ const root = await mkdtemp(join(tmpdir(), "laya-ts-audit-"));+ try {+ await mkdir(join(root, "tokenizer"));+ await writeFile(join(root, "rl_agent_config.json"), JSON.stringify({ max_len: 32 }));+ await writeFile(join(root, "tokenizer", "tokenizer.json"), JSON.stringify({ model: { vocab: {}, merges: [] } }));+ const bundle = await loadNodeBundle(root);+ expect(bundle.tokenizerJson).toEqual({ model: { vocab: {}, merges: [] } });+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ });+});diff --git a/laya/agent.py b/laya/agent.pyindex ab3b3f8b..b0fb78a7 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -205,6 +205,8 @@ class Agent(HookRegistry):# without it (for example a hand-constructed runtime in tests).amp_enabled = Falsemps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT+ # The stock forward is also used by lightweight runtimes built with __new__ in tests.+ _fast = Nonedef __init__(self,@@ -598,6 +600,13 @@ class Agent(HookRegistry):"""Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""use_amp = self._amp_enabled_for(b["input_ids"].shape[0])+ if self._fast is not None and b["input_ids"].shape[1] > self._fast.max_len:+ raise ValueError(+ "the CUDA fast path was built for max_len=%d; this request needs %d tokens. "+ "Use max_len <= %d or load without fast=True."+ % (self._fast.max_len, b["input_ids"].shape[1], self._fast.max_len)+ )+def run():# Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes# effect on the retry. A disabled gate never enters torch.autocast at all.@@ -617,10 +626,16 @@ class Agent(HookRegistry):low = str(e).lower()if self.device.type != "cpu" and ("memory" in low or "cuda" in low):print("Warning: GPU memory exceeded during inference. Falling back to CPU...")+ # FastLaya keeps copied CUDA weights and replaces model.forward. Move+ # the model first without that replacement, or the retry would still+ # execute on the failed CUDA fast path.+ self.deaccelerate()self.device = torch.device("cpu")self.dtype = torch.float32self.amp_enabled = Falseself.model.to(self.device)+ if torch.cuda.is_available():+ torch.cuda.empty_cache()return run()if use_amp and self.device.type in ("mps", "cpu"):# Not every MPS/CPU build implements autocast for every op. Drop to fulldiff --git a/laya/cli.py b/laya/cli.pyindex 21b199fe..31cbee3b 100644--- a/laya/cli.py+++ b/laya/cli.py@@ -67,7 +67,9 @@ def show_answers(result):print()for qid, answer in result.get("answers", {}).items():if "choice" in answer:- detail = "%s (p=%.3f)" % (answer["choice"], answer.get("probability", 0.0))+ choice = answer["choice"]+ probability = answer.get("probabilities", {}).get(choice, 0.0)+ detail = "%s (p=%.3f)" % (choice, probability)elif "score" in answer:detail = "%.2f" % answer["score"]elif "noul" in answer:diff --git a/laya/fast.py b/laya/fast.pyindex f08b5967..c960134c 100644--- a/laya/fast.py+++ b/laya/fast.py@@ -5,12 +5,21 @@Requires CUDA and `pip install laya[fast]` (tilelang). Falls back to the stock forward otherwise."""import sys+import threadingimport torchfrom . import tl_kernels as KBF = torch.bfloat16+def _top_two(probs):+ """Return two columns for confidence features, including k == 1."""+ if probs.shape[-1] == 1:+ top1 = probs[:, 0]+ return torch.stack([top1, torch.zeros_like(top1)], dim=-1)+ return probs.topk(2, -1).values++def _bucket_n(n):return 1 << max(0, (n - 1).bit_length())@@ -23,6 +32,10 @@ class FastLaya:dev = next(model.parameters()).deviceself.dev = devself.use_graphs = use_graphs+ # CUDA graphs reuse static input/output buffers. Keep the complete+ # forward under one lock so concurrent callers cannot overwrite those+ # buffers between replay and head decoding.+ self._forward_lock = threading.RLock()self.verbose = verboseself.H, self.Dh, self.D = cfg.num_attention_heads, cfg.hidden_size // cfg.num_attention_heads, cfg.hidden_sizeself.F = cfg.intermediate_size@@ -169,6 +182,11 @@ class FastLaya:# ------------------------------------------------------------------ DecisionModel.forward replacement@torch.no_grad()def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):+ with self._forward_lock:+ return self._forward_unlocked(input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder)++ @torch.no_grad()+ def _forward_unlocked(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):m = self.mN, L0 = input_ids.shapeg = 16 if L0 <= self.DYNAMIC_MAX_L else self.LONG_BUCKET@@ -190,7 +208,10 @@ class FastLaya:p = torch.softmax(logits, -1)k = marker_mask.sum(-1).clamp(min=2).float()ent = -(p * torch.log(p.clamp_min(1e-9))).sum(-1) / torch.log(k)- top2 = p.topk(2, -1).values+ # Choice questions are allowed to contain one criterion. The stock+ # DecisionModel handles that case, but topk(2) raises when the marker+ # dimension has width one.+ top2 = _top_two(p)feats = torch.stack([top2[:, 0], top2[:, 0] - top2[:, 1], ent, k / 255.0], -1)pooled = h[:, 0].float()act_logits = m.act_head(torch.cat([pooled, feats], -1))diff --git a/laya/integrations/langchain.py b/laya/integrations/langchain.pyindex 4c85a2f6..f7e5cb9b 100644--- a/laya/integrations/langchain.py+++ b/laya/integrations/langchain.py@@ -9,7 +9,9 @@ deployments (your own `laya-serve`) without requiring PyTorch on edge clients.from __future__ import annotationsimport json+import threadingimport urllib.error+import urllib.parseimport urllib.requestfrom typing import Any, Callable, Dict, Optional, Sequence, Union@@ -77,6 +79,23 @@ def _extract_from_messages_list(msgs: Sequence[Any]) -> str:return str(getattr(last, "content", last))+class _SameOriginRedirectHandler(urllib.request.HTTPRedirectHandler):+ """Do not forward bearer credentials across an origin or HTTPS downgrade."""++ def redirect_request(self, req, fp, code, msg, headers, newurl):+ old = urllib.parse.urlsplit(req.full_url)+ new = urllib.parse.urlsplit(newurl)+ old_port = old.port or (443 if old.scheme.lower() == "https" else 80)+ new_port = new.port or (443 if new.scheme.lower() == "https" else 80)+ if (+ old.scheme.lower() != new.scheme.lower()+ or (old.hostname or "").lower() != (new.hostname or "").lower()+ or old_port != new_port+ ):+ raise urllib.error.URLError("refusing cross-origin or HTTPS-downgrade redirect")+ return super().redirect_request(req, fp, code, msg, headers, newurl)++def _call_remote(base_url: str,state: Any,@@ -100,8 +119,9 @@ def _call_remote(headers["Authorization"] = f"Bearer {api_key}"req = urllib.request.Request(url, data=data, headers=headers, method="POST")+ opener = urllib.request.build_opener(_SameOriginRedirectHandler())try:- with urllib.request.urlopen(req, timeout=timeout) as resp:+ with opener.open(req, timeout=timeout) as resp:return json.loads(resp.read().decode("utf-8"))except urllib.error.HTTPError as e:body = e.read().decode("utf-8", errors="replace")@@ -111,13 +131,16 @@ def _call_remote(_DEFAULT_ROUTER = None+_DEFAULT_ROUTER_LOCK = threading.Lock()def _get_default_router():global _DEFAULT_ROUTERif _DEFAULT_ROUTER is None:- from ..router import Router- _DEFAULT_ROUTER = Router()+ with _DEFAULT_ROUTER_LOCK:+ if _DEFAULT_ROUTER is None:+ from ..router import Router+ _DEFAULT_ROUTER = Router()return _DEFAULT_ROUTERdiff --git a/laya/mcp/server.py b/laya/mcp/server.pyindex b3b67602..eea0847a 100644--- a/laya/mcp/server.py+++ b/laya/mcp/server.py@@ -19,6 +19,7 @@ from __future__ import annotationsimport jsonimport osimport sys+import threadingfrom importlib import metadata as _metadatafrom typing import Any@@ -54,6 +55,7 @@ except Exception: # running from a source checkout without install metadataserver = MCPServer("laya", version=_LAYA_VERSION)_ROUTER: Any = None+_ROUTER_LOCK = threading.Lock()# MCP default preload list. laya.serve preloads every checkpoint when LAYA_MODELS# is empty; MCP keeps typed-decisions lazy on purpose (it is ~as big as the other@@ -95,19 +97,22 @@ def _ensure_router() -> Any:global _ROUTERif _ROUTER is not None:return _ROUTER- try:- from laya import Router- except Exception as exc:- raise ToolError("internal_error", f"cannot import laya: {exc}") from exc- try:- _apply_thread_limit()- router = Router(device=env_device())- if _env_bool("LAYA_PRELOAD", True):- router.preload(_models_from_env())- except Exception as exc:- raise ToolError("internal_error", f"router construction failed: {exc}") from exc- _ROUTER = router- return router+ with _ROUTER_LOCK:+ if _ROUTER is not None:+ return _ROUTER+ try:+ from laya import Router+ except Exception as exc:+ raise ToolError("internal_error", f"cannot import laya: {exc}") from exc+ try:+ _apply_thread_limit()+ router = Router(device=env_device())+ if _env_bool("LAYA_PRELOAD", True):+ router.preload(_models_from_env())+ except Exception as exc:+ raise ToolError("internal_error", f"router construction failed: {exc}") from exc+ _ROUTER = router+ return routerdef _dump(payload: Any) -> str:diff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex 58e3ff2c..b7f2e2d6 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -100,7 +100,11 @@ class ONNXAgent(HookRegistry):f"ONNX model not found at {onnx_path!r}. Please run export_onnx.py first.")- # Load Tokenizer+ # Load Tokenizer. Keep this compatibility fix in sync with Agent: checkpoints+ # produced by newer Transformers versions can contain TokenizersBackend or a list-valued+ # extra_special_tokens field that older loaders cannot parse.+ from .agent import _fix_tokenizer_config+ _fix_tokenizer_config(model_dir)tok_dir = os.path.join(model_dir, "tokenizer")self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))@@ -111,7 +115,19 @@ class ONNXAgent(HookRegistry):if "CUDAExecutionProvider" in availableelse ["CPUExecutionProvider"])- self.session = ort.InferenceSession(onnx_path, providers=providers)+ try:+ self.session = ort.InferenceSession(onnx_path, providers=providers)+ except Exception:+ # A provider can be listed by onnxruntime yet still fail to initialize+ # (missing CUDA libraries, unsupported driver, mismatched DLLs). Keep+ # the CPU runtime usable instead of making model construction fail.+ if providers == ["CPUExecutionProvider"]:+ raise+ warnings.warn(+ "laya ONNX: CUDAExecutionProvider initialization failed; falling back to CPUExecutionProvider.",+ RuntimeWarning, stacklevel=2,+ )+ self.session = ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})@@ -198,6 +214,12 @@ class ONNXAgent(HookRegistry):from .agent import Agent as _Agentids = list(questions.keys())+ if not ids:+ return {+ "model": "laya-rl-agent-onnx",+ "answers": {},+ "usage": {"input_tokens": 0, "output_tokens": 0},+ }for qid in ids:_Agent._check_question(qid, questions[qid])items = []@@ -206,7 +228,10 @@ class ONNXAgent(HookRegistry):for qid in ids:q = self._to_internal(questions[qid])- seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len)+ seq, markers = build_sequence(+ self.tok, state, q, max_len, head_max_len,+ truncate_left=isinstance(state, list),+ )if len(markers) != len(render_options(q)):raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})diff --git a/laya/serve.py b/laya/serve.pyindex 94e3a660..c0fc1bc3 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -35,6 +35,7 @@ and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.import hmacimport jsonimport os+from contextlib import asynccontextmanagerfrom typing import Any, Dict, Optional# The three checkpoint names the router understands; used to decide whether a@@ -197,9 +198,19 @@ def create_app(router: Optional[Any] = None):# exists (module scope, TestClient startup, a preload script).gate: Optional[asyncio.Lock] = None+ @asynccontextmanager+ async def lifespan(_app: FastAPI):+ try:+ yield+ finally:+ # TestClient, embedded ASGI apps, and process supervisors all need+ # the executor to drain when the app stops.+ pool.shutdown(wait=True, cancel_futures=True)+app = FastAPI(title="laya-serve",summary="Laya System-1 decisions over the TypeSafe Jev /v1/systemone protocol",+ lifespan=lifespan,)# Compared as bytes, not str. `hmac.compare_digest` raises TypeError when a strdiff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.pynew file mode 100644index 00000000..3c90c0b3--- /dev/null+++ b/tests/test_audit_regressions.py@@ -0,0 +1,210 @@+"""Regression tests for fixes found during the cross-runtime audit."""+import io+import json+from contextlib import redirect_stdout++from laya.cli import show_answers+from laya.integrations.langchain import _SameOriginRedirectHandler+from laya.onnx_agent import ONNXAgent+from laya.serve import MAX_BODY_BYTES, create_app+++class FakeRouter:+ loaded = ["english"]++ def predict(self, state, questions, model=None):+ return {+ "model": "test",+ "answers": {},+ "usage": {"input_tokens": 0, "output_tokens": 0},+ }+++def test_serve_rejects_oversized_stream_without_content_length():+ from fastapi.testclient import TestClient++ app = create_app(router=FakeRouter())+ client = TestClient(app)+ payload = {+ "state": "hello",+ "questions": {"q": {"type": "noul", "instructions": "?"}},+ "padding": "x" * (MAX_BODY_BYTES + 1024),+ }+ encoded = json.dumps(payload).encode()+ # An iterator makes httpx use a streaming body, so this exercises the+ # Content-Length-independent path rather than the existing header guard.+ response = client.post(+ "/v1/systemone",+ content=iter([encoded[:32], encoded[32:]]),+ headers={"content-type": "application/json"},+ )+ assert response.status_code == 413+++def test_serve_lifespan_shuts_down_inference_executor():+ from fastapi.testclient import TestClient+ from unittest.mock import patch++ with patch("concurrent.futures.ThreadPoolExecutor.shutdown") as shutdown:+ with TestClient(create_app(router=FakeRouter())):+ pass+ shutdown.assert_called_once_with(wait=True, cancel_futures=True)+++def test_serve_non_ascii_auth_is_401_not_500(monkeypatch):+ from fastapi.testclient import TestClient++ monkeypatch.setenv("LAYA_API_KEY", "s3cret")+ app = create_app(router=FakeRouter())+ client = TestClient(app)+ response = client.post(+ "/v1/systemone",+ content=b'{"state":"hello","questions":{}}',+ headers=[+ ("content-type", "application/json"),+ (b"authorization", b"Bearer caf\xe9"),+ ],+ )+ assert response.status_code == 401+++def test_onnx_empty_questions_matches_agent_contract():+ agent = ONNXAgent.__new__(ONNXAgent)+ result = ONNXAgent._infer(agent, "hello", {})+ assert result == {+ "model": "laya-rl-agent-onnx",+ "answers": {},+ "usage": {"input_tokens": 0, "output_tokens": 0},+ }+++def test_onnx_long_list_state_keeps_newest_turn(monkeypatch):+ import laya.onnx_agent as module++ seen = []+ original = module.build_sequence++ def fake_build(*args, **kwargs):+ seen.append(kwargs.get("truncate_left"))+ raise RuntimeError("stop after encoding")++ monkeypatch.setattr(module, "build_sequence", fake_build)+ agent = ONNXAgent.__new__(ONNXAgent)+ agent.cfg = {"max_len": 32, "head_max_len": 16}+ agent.temperature = [1.0, 1.0, 1.0]+ agent.temperature_by_options = {}+ agent.tok = object()+ try:+ ONNXAgent._infer(agent, [{"text": "old"}, {"text": "new"}], {+ "q": {"type": "noul", "instructions": "?"}+ })+ except RuntimeError as exc:+ assert str(exc) == "stop after encoding"+ else:+ raise AssertionError("encoding should have been reached")+ assert seen == [True]+++def test_cli_renders_choice_probability_from_probabilities_map():+ output = io.StringIO()+ with redirect_stdout(output):+ show_answers({+ "answers": {+ "q": {"type": "choice", "choice": "yes", "probabilities": {"yes": 0.91, "no": 0.09}}+ }+ })+ assert "yes (p=0.910)" in output.getvalue()+++def test_default_langchain_router_initialization_is_singleton(monkeypatch):+ import threading++ import laya.integrations.langchain as module+ import laya.router as router_module++ created = []+ lock = threading.Lock()++ class FakeRouter:+ def __init__(self):+ with lock:+ created.append(self)++ monkeypatch.setattr(router_module, "Router", FakeRouter)+ monkeypatch.setattr(module, "_DEFAULT_ROUTER", None)+ barrier = threading.Barrier(2)+ results = []++ def get_router():+ barrier.wait()+ results.append(module._get_default_router())++ threads = [threading.Thread(target=get_router) for _ in range(2)]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join()+ assert len(created) == 1+ assert results[0] is results[1]+++def test_remote_redirect_handler_rejects_cross_origin_credentials():+ import urllib.error+ import urllib.request++ request = urllib.request.Request(+ "https://laya.example/v1/systemone",+ headers={"Authorization": "Bearer secret"},+ )+ handler = _SameOriginRedirectHandler()+ same_origin = handler.redirect_request(+ request, None, 302, "Found", {}, "https://laya.example/v1/systemone?retry=1"+ )+ assert same_origin.get_header("Authorization") == "Bearer secret"+ try:+ handler.redirect_request(+ request, None, 302, "Found", {}, "http://attacker.example/collect"+ )+ except urllib.error.URLError as exc:+ assert "cross-origin" in str(exc)+ else:+ raise AssertionError("cross-origin redirect was accepted")+++def test_fast_path_rejects_sequences_over_its_capacity():+ from types import SimpleNamespace++ import torch++ from laya.agent import Agent++ agent = Agent.__new__(Agent)+ agent._fast = SimpleNamespace(max_len=16)+ agent.device = torch.device("cpu")+ agent.dtype = torch.float32+ agent.amp_enabled = False+ agent.mps_amp_min_rows = 5+ batch = {+ "input_ids": torch.zeros((1, 17), dtype=torch.long),+ "attention_mask": torch.ones((1, 17), dtype=torch.long),+ "marker_pos": torch.zeros((1, 2), dtype=torch.long),+ "marker_mask": torch.ones((1, 2), dtype=torch.bool),+ "qtype": torch.zeros((1,), dtype=torch.long),+ }+ try:+ agent._infer(batch)+ except ValueError as exc:+ assert "max_len=16" in str(exc)+ else:+ raise AssertionError("oversized fast-path request was accepted")+++def test_fast_top_two_handles_one_option():+ import pytest+ pytest.importorskip("tilelang")+ from laya.fast import _top_two+ import torch++ values = _top_two(torch.tensor([[0.75]]))+ assert values.shape == (1, 2)+ assert torch.allclose(values, torch.tensor([[0.75, 0.0]]))