Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix: harden inference and runtime parity (#319)

Kunj SHAHd0aeae31256d
diff --git a/examples/server.py b/examples/server.py
index 32324fcc..5214a21f 100644
--- a/examples/server.py
+++ b/examples/server.py
@@ -37,6 +37,7 @@ from typing import Any, Dict, List, Optional, Union
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
+from starlette.concurrency import run_in_threadpool
from pydantic import BaseModel, Field, field_validator, model_validator
from pydantic import ValidationError as PydanticValidationError
@@ -1054,8 +1055,9 @@ async def gui_predict(request: Request) -> HTMLResponse:
questions = _questions(req.questions)
try:
started = time.perf_counter()
- res = _predict(
- req.state, questions, model=req.model, task=req.task, lang=req.lang
+ res = await run_in_threadpool(
+ _predict, req.state, questions,
+ model=req.model, task=req.task, lang=req.lang,
)
res["_elapsed"] = time.perf_counter() - started
except Exception as exc:
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 49a65cd8..11a33db7 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -145,10 +145,37 @@ export function checkQuestion(qid: string, qdef: unknown): void {
throw new Error(
`question ${qidStr(qid)}: a noul question takes 'criteria' as a dict with optional 'true'/'false' descriptions, or omits it`,
);
+ } else if (crit && typeof crit === "object" && !Array.isArray(crit)) {
+ const invalid = Object.keys(crit as Record<string, unknown>).filter((key) => key !== "true" && key !== "false");
+ if (invalid.length > 0) {
+ throw new Error(
+ `question ${qidStr(qid)}: noul criteria may contain only 'true' and 'false'; got ${JSON.stringify(invalid)}`,
+ );
+ }
+ }
+ if ("labels" in q && t !== "noul") {
+ throw new Error(`question ${qidStr(qid)}: 'labels' is only supported for noul questions`);
+ }
+ if (t === "noul" && "labels" in q) {
+ const labels = q["labels"];
+ if (typeof labels !== "object" || labels === null || Array.isArray(labels)) {
+ throw new Error(`question ${qidStr(qid)}: noul labels must be an object with 'false' and 'true'`);
+ }
+ const entries = Object.entries(labels as Record<string, unknown>);
+ const keys = entries.map(([key]) => key).sort();
+ if (keys.length !== 2 || keys[0] !== "false" || keys[1] !== "true" ||
+ entries.some(([, value]) => typeof value !== "string" || value.trim() === "")) {
+ throw new Error(`question ${qidStr(qid)}: noul labels must map exactly 'false' and 'true' to distinct non-empty strings`);
+ }
+ const falseLabel = String((labels as Record<string, unknown>)["false"]).trim();
+ const trueLabel = String((labels as Record<string, unknown>)["true"]).trim();
+ if (falseLabel === trueLabel) {
+ throw new Error(`question ${qidStr(qid)}: noul labels must be distinct`);
+ }
}
}
-export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul"; ins: string; crit: unknown } {
+export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } {
const t = qdef["type"] as "choice" | "score" | "noul";
let crit: unknown = qdef["criteria"];
if (t === "choice" && Array.isArray(crit)) {
@@ -157,8 +184,15 @@ export function toInternal(qdef: QuestionDef): { t: "choice" | "score" | "noul";
crit = Object.fromEntries(Object.entries(crit as Record<string, unknown>).map(([k, v]) => [String(k).toLowerCase(), v]));
}
let ins: unknown = qdef["instructions"];
- if (typeof ins !== "string") ins = JSON.stringify(ins);
- return { t, ins: ins as string, crit };
+ if (typeof ins !== "string") ins = serializeState(ins);
+ const out: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } } = {
+ t, ins: ins as string, crit,
+ };
+ if (t === "noul" && qdef["labels"] && typeof qdef["labels"] === "object" && !Array.isArray(qdef["labels"])) {
+ const labels = qdef["labels"] as Record<string, unknown>;
+ out.labels = { false: String(labels["false"]).trim(), true: String(labels["true"]).trim() };
+ }
+ return out;
}
export function defaultTokenizer(): TokenizerLike {
@@ -190,27 +224,6 @@ function tokenizerFromHF(tokenizerJson: unknown): TokenizerLike | null {
};
}
-/** Fallback when tokenizer.json is absent: ids from rl_agent_config, if present. */
-function tokenizerFromConfig(cfg: AgentCfg): TokenizerLike | null {
- const c = cfg as Record<string, unknown>;
- const num = (v: unknown): number | null =>
- typeof v === "number" && Number.isFinite(v) ? v : null;
- const cls = num(c["cls_token_id"]) ?? num(c["cls_id"]);
- const sep = num(c["sep_token_id"]) ?? num(c["sep_id"]);
- const mask = num(c["mask_token_id"]) ?? num(c["mask_id"]);
- const pad = num(c["pad_token_id"]) ?? num(c["pad_id"]);
- if (cls === null && sep === null && mask === null && pad === null) return null;
- const d = defaultTokenizer();
- return {
- clsId: cls ?? d.clsId,
- sepId: sep ?? d.sepId,
- maskId: mask ?? d.maskId,
- padId: pad ?? d.padId,
- maskToken: typeof c["mask_token"] === "string" ? (c["mask_token"] as string) : d.maskToken,
- encode: d.encode,
- };
-}
-
const r4 = (v: number): number => Math.round(v * 1e4) / 1e4;
export class Agent extends HookRegistry {
@@ -341,7 +354,7 @@ export class Agent extends HookRegistry {
return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } };
}
const items: { ids: number[]; markers: number[]; qtype: number }[] = [];
- const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[] = [];
+ const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }[] = [];
// The state text is shared by every question and is usually the longest text in the
// sequence — encode it once and compose the per-question prefixes onto it.
const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));
@@ -350,7 +363,9 @@ export class Agent extends HookRegistry {
const q = toInternal(questions[qid]);
internals.push(q);
const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen);
- const { ids: seq, markers } = sequenceWithState(prefix, stAll, this.tok.sepId, this.maxLen);
+ const { ids: seq, markers } = sequenceWithState(
+ prefix, stAll, this.tok.sepId, this.maxLen, Array.isArray(state),
+ );
if (markers.length !== renderOptions(q).length) {
throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`);
}
@@ -362,6 +377,16 @@ export class Agent extends HookRegistry {
const nTokens = batch.attentionMask.flat().reduce((a, b) => a + b, 0);
const { lastHidden } = await this.provider.runEncoder(batch);
const { logits, act } = await this.provider.runHead(lastHidden, batch);
+ if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < ids.length || act.length < ids.length) {
+ throw new Error("model provider returned fewer output rows than input items");
+ }
+ for (let r = 0; r < ids.length; r++) {
+ const k = items[r].markers.length;
+ if (!Array.isArray(logits[r]) || logits[r].length < k || !Array.isArray(act[r]) || act[r].length === 0 ||
+ logits[r].some((v) => !Number.isFinite(v)) || act[r].some((v) => !Number.isFinite(v))) {
+ throw new Error(`model provider returned invalid output for question ${ids[r]}`);
+ }
+ }
const answers: Record<string, SystemAnswer> = {};
for (let r = 0; r < ids.length; r++) {
@@ -454,12 +479,21 @@ export class Agent extends HookRegistry {
dir = bundle.dir;
provider = await createNodeProvider(dir, { device: opts?.device, numThreads: opts?.numThreads });
}
- let tok: TokenizerLike | null = null;
+ if (!tokenizerJson) {
+ throw new Error(
+ `Incompatible model: tokenizer.json is missing or invalid in ${JSON.stringify(dir)}`,
+ );
+ }
+ let tok: TokenizerLike;
try {
- tok = tokenizerJson ? tokenizerFromHF(tokenizerJson) : tokenizerFromConfig(cfg);
- } catch {
- tok = null;
+ const parsed = tokenizerFromHF(tokenizerJson);
+ if (!parsed) throw new Error("unsupported tokenizer.json format");
+ tok = parsed;
+ } catch (error) {
+ throw new Error(
+ `Incompatible model: tokenizer.json is missing or invalid in ${JSON.stringify(dir)}: ${String(error)}`,
+ );
}
- return new Agent({ provider, tok: tok ?? defaultTokenizer(), cfg });
+ return new Agent({ provider, tok, cfg });
}
}
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index f0b176a7..f46f1b51 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -1,11 +1,17 @@
import type { TokenizerLike } from "./tokenizer.js";
export type QType = "choice" | "score" | "noul";
-export interface InternalQ { t: QType; ins: string; crit: unknown }
+export interface InternalQ { t: QType; ins: string; crit: unknown; labels?: { false: string; true: string } }
/** Python `json.dumps(v, ensure_ascii=False)` replica: separators (", ", ": "),
unicode raw, unknown types fall back to undefined (caller applies str()). */
function pyJson(v: unknown): string | undefined {
if (v === null) return "null";
- if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") return JSON.stringify(v);
+ if (typeof v === "string" || typeof v === "boolean") return JSON.stringify(v);
+ if (typeof v === "number") {
+ if (Number.isNaN(v)) return "NaN";
+ if (v === Infinity) return "Infinity";
+ if (v === -Infinity) return "-Infinity";
+ return JSON.stringify(v);
+ }
if (Array.isArray(v)) return `[${v.map((x) => pyJson(x) ?? "null").join(", ")}]`;
if (typeof v === "object") {
const proto = Object.getPrototypeOf(v);
@@ -36,10 +42,11 @@ export function renderOptions(q: InternalQ): string[] {
return (q.crit as unknown[]).map((c, i) => `level ${i}: ${renderCriterion(c)}`);
}
const crit = (q.crit ?? {}) as Record<string, unknown>;
+ const labels = q.labels ?? { false: "false", true: "true" };
const f = crit["false"], t = crit["true"];
return [
- "false: " + (f !== null && f !== undefined && f !== "" ? renderCriterion(f) : "no, the statement does not hold"),
- "true: " + (t !== null && t !== undefined && t !== "" ? renderCriterion(t) : "yes, the statement holds"),
+ labels.false + ": " + (f !== null && f !== undefined && f !== "" ? renderCriterion(f) : "no, the statement does not hold"),
+ labels.true + ": " + (t !== null && t !== undefined && t !== "" ? renderCriterion(t) : "yes, the statement holds"),
];
}
export interface QuestionPrefix {
@@ -102,6 +109,7 @@ export function confidenceFromProbs(p: number[]): number {
}
export const TEMP_MIN = 0.5, TEMP_MAX = 5.0;
export function clampTemperature(t: unknown): number {
+ if (t === null || t === undefined || t === "" || typeof t === "boolean") return 1.0;
const f = typeof t === "number" ? t : Number(t);
if (!Number.isFinite(f)) return 1.0;
return Math.min(TEMP_MAX, Math.max(TEMP_MIN, f));
diff --git a/laya-ts/src/providers.ts b/laya-ts/src/providers.ts
index 91084b08..01670908 100644
--- a/laya-ts/src/providers.ts
+++ b/laya-ts/src/providers.ts
@@ -195,7 +195,7 @@ export async function loadNodeBundle(
await fs.mkdir(cache, { recursive: true });
const token =
opts?.token ?? (typeof process !== "undefined" ? (process as any).env?.["HF_TOKEN"] : undefined);
- for (const f of ["rl_agent_config.json", "tokenizer.json", "encoder.onnx", "head.onnx"]) {
+ for (const f of ["rl_agent_config.json", "tokenizer.json", "tokenizer/tokenizer.json", "encoder.onnx", "head.onnx"]) {
try {
await fs.stat(path.join(cache, f));
} catch {
@@ -209,7 +209,9 @@ export async function loadNodeBundle(
}
continue;
}
- await fs.writeFile(path.join(cache, f), new Uint8Array(await res.arrayBuffer()));
+ const target = path.join(cache, f);
+ await fs.mkdir(path.dirname(target), { recursive: true });
+ await fs.writeFile(target, new Uint8Array(await res.arrayBuffer()));
}
}
dir = cache;
@@ -223,10 +225,13 @@ export async function loadNodeBundle(
);
}
let tokenizerJson: unknown | null = null;
- try {
- tokenizerJson = JSON.parse(await fs.readFile(path.join(dir, "tokenizer.json"), "utf8"));
- } catch {
- tokenizerJson = null;
+ for (const candidate of ["tokenizer.json", "tokenizer/tokenizer.json"]) {
+ try {
+ tokenizerJson = JSON.parse(await fs.readFile(path.join(dir, candidate), "utf8"));
+ break;
+ } catch {
+ // Try the next supported Hugging Face layout.
+ }
}
return { dir, cfg, tokenizerJson };
}
@@ -257,10 +262,13 @@ export async function loadWebBundle(
throw new Error(`Incompatible model: ${JSON.stringify(repoOrUrl)} does not contain 'rl_agent_config.json'.`);
}
let tokenizerJson: unknown | null = null;
- try {
- tokenizerJson = await fetchJson(`${base}/tokenizer.json`);
- } catch {
- tokenizerJson = null;
+ for (const candidate of ["tokenizer.json", "tokenizer/tokenizer.json"]) {
+ try {
+ tokenizerJson = await fetchJson(`${base}/${candidate}`);
+ break;
+ } catch {
+ // Try the next supported Hugging Face layout.
+ }
}
return { dir: base, cfg, tokenizerJson };
}
diff --git a/laya-ts/tests/audit-regressions.test.ts b/laya-ts/tests/audit-regressions.test.ts
new file mode 100644
index 00000000..fb3120c6
--- /dev/null
+++ b/laya-ts/tests/audit-regressions.test.ts
@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Agent, checkQuestion, toInternal } from "../src/agent.js";
+import { buildSequence } from "../src/common.js";
+import { loadNodeBundle } from "../src/providers.js";
+
+const tokenizer = () => ({
+ clsId: 1,
+ sepId: 2,
+ maskId: 3,
+ padId: 0,
+ maskToken: "[MASK]",
+ encode(text: string): number[] {
+ return Array.from(text).map((c) => c.codePointAt(0) ?? 0);
+ },
+});
+
+const provider = (calls: any[] = []) => ({
+ async runEncoder(batch: any) {
+ calls.push(batch);
+ return { lastHidden: [[[0, 0]]] };
+ },
+ async runHead(_hidden: any, _batch: any) {
+ return { logits: [[1, 0]], act: [[1, 0]] };
+ },
+});
+
+describe("audit regressions", () => {
+ it("preserves the newest conversation turn", () => {
+ const q = toInternal({ type: "noul", instructions: "?" });
+ const result = buildSequence(tokenizer(), [
+ { text: "old context that must be truncated" },
+ { text: "newest refund request" },
+ ], q, 64, 32, undefined, true);
+ const rendered = result.ids.map((id) => String.fromCodePoint(id)).join("");
+ expect(rendered).toContain("refund request");
+ });
+
+ it("renders custom noul labels and validates unsupported labels", () => {
+ const q = toInternal({
+ type: "noul",
+ instructions: "?",
+ criteria: {},
+ labels: { false: "No", true: "Yes" },
+ });
+ const sequence = buildSequence(tokenizer(), "state", q, 64, 32);
+ const rendered = sequence.ids.map((id) => String.fromCodePoint(id)).join("");
+ expect(rendered).toContain("No");
+ expect(rendered).toContain("Yes");
+ expect(() => checkQuestion("q", { type: "noul", instructions: "?", labels: { false: "No" } })).toThrow("labels");
+ expect(() => checkQuestion("q", { type: "choice", instructions: "?", criteria: ["yes"], labels: { false: "No", true: "Yes" } })).toThrow("labels");
+ });
+
+ it("rejects malformed provider output", async () => {
+ const agent = new Agent({ provider: {
+ async runEncoder(_batch: any) { return { lastHidden: [[[0, 0]]] }; },
+ async runHead(_hidden: any, _batch: any) { return { logits: [[]], act: [[1, 0]] }; },
+ }, tok: tokenizer() } as any);
+ await expect(agent.systemOne("state", { q: { type: "noul", instructions: "?" } })).rejects.toThrow("invalid output");
+ });
+
+ it("loads the nested Hugging Face tokenizer layout", async () => {
+ const root = await mkdtemp(join(tmpdir(), "laya-ts-audit-"));
+ try {
+ await mkdir(join(root, "tokenizer"));
+ await writeFile(join(root, "rl_agent_config.json"), JSON.stringify({ max_len: 32 }));
+ await writeFile(join(root, "tokenizer", "tokenizer.json"), JSON.stringify({ model: { vocab: {}, merges: [] } }));
+ const bundle = await loadNodeBundle(root);
+ expect(bundle.tokenizerJson).toEqual({ model: { vocab: {}, merges: [] } });
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/laya/agent.py b/laya/agent.py
index ab3b3f8b..b0fb78a7 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -205,6 +205,8 @@ class Agent(HookRegistry):
# without it (for example a hand-constructed runtime in tests).
amp_enabled = False
mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT
+ # The stock forward is also used by lightweight runtimes built with __new__ in tests.
+ _fast = None
def __init__(
self,
@@ -598,6 +600,13 @@ class Agent(HookRegistry):
"""Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""
use_amp = self._amp_enabled_for(b["input_ids"].shape[0])
+ if self._fast is not None and b["input_ids"].shape[1] > self._fast.max_len:
+ raise ValueError(
+ "the CUDA fast path was built for max_len=%d; this request needs %d tokens. "
+ "Use max_len <= %d or load without fast=True."
+ % (self._fast.max_len, b["input_ids"].shape[1], self._fast.max_len)
+ )
+
def run():
# Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes
# effect on the retry. A disabled gate never enters torch.autocast at all.
@@ -617,10 +626,16 @@ class Agent(HookRegistry):
low = str(e).lower()
if self.device.type != "cpu" and ("memory" in low or "cuda" in low):
print("Warning: GPU memory exceeded during inference. Falling back to CPU...")
+ # FastLaya keeps copied CUDA weights and replaces model.forward. Move
+ # the model first without that replacement, or the retry would still
+ # execute on the failed CUDA fast path.
+ self.deaccelerate()
self.device = torch.device("cpu")
self.dtype = torch.float32
self.amp_enabled = False
self.model.to(self.device)
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
return run()
if use_amp and self.device.type in ("mps", "cpu"):
# Not every MPS/CPU build implements autocast for every op. Drop to full
diff --git a/laya/cli.py b/laya/cli.py
index 21b199fe..31cbee3b 100644
--- a/laya/cli.py
+++ b/laya/cli.py
@@ -67,7 +67,9 @@ def show_answers(result):
print()
for qid, answer in result.get("answers", {}).items():
if "choice" in answer:
- detail = "%s (p=%.3f)" % (answer["choice"], answer.get("probability", 0.0))
+ choice = answer["choice"]
+ probability = answer.get("probabilities", {}).get(choice, 0.0)
+ detail = "%s (p=%.3f)" % (choice, probability)
elif "score" in answer:
detail = "%.2f" % answer["score"]
elif "noul" in answer:
diff --git a/laya/fast.py b/laya/fast.py
index f08b5967..c960134c 100644
--- a/laya/fast.py
+++ b/laya/fast.py
@@ -5,12 +5,21 @@
Requires CUDA and `pip install laya[fast]` (tilelang). Falls back to the stock forward otherwise.
"""
import sys
+import threading
import torch
from . import tl_kernels as K
BF = torch.bfloat16
+def _top_two(probs):
+ """Return two columns for confidence features, including k == 1."""
+ if probs.shape[-1] == 1:
+ top1 = probs[:, 0]
+ return torch.stack([top1, torch.zeros_like(top1)], dim=-1)
+ return probs.topk(2, -1).values
+
+
def _bucket_n(n):
return 1 << max(0, (n - 1).bit_length())
@@ -23,6 +32,10 @@ class FastLaya:
dev = next(model.parameters()).device
self.dev = dev
self.use_graphs = use_graphs
+ # CUDA graphs reuse static input/output buffers. Keep the complete
+ # forward under one lock so concurrent callers cannot overwrite those
+ # buffers between replay and head decoding.
+ self._forward_lock = threading.RLock()
self.verbose = verbose
self.H, self.Dh, self.D = cfg.num_attention_heads, cfg.hidden_size // cfg.num_attention_heads, cfg.hidden_size
self.F = cfg.intermediate_size
@@ -169,6 +182,11 @@ class FastLaya:
# ------------------------------------------------------------------ DecisionModel.forward replacement
@torch.no_grad()
def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):
+ with self._forward_lock:
+ return self._forward_unlocked(input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder)
+
+ @torch.no_grad()
+ def _forward_unlocked(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):
m = self.m
N, L0 = input_ids.shape
g = 16 if L0 <= self.DYNAMIC_MAX_L else self.LONG_BUCKET
@@ -190,7 +208,10 @@ class FastLaya:
p = torch.softmax(logits, -1)
k = marker_mask.sum(-1).clamp(min=2).float()
ent = -(p * torch.log(p.clamp_min(1e-9))).sum(-1) / torch.log(k)
- top2 = p.topk(2, -1).values
+ # Choice questions are allowed to contain one criterion. The stock
+ # DecisionModel handles that case, but topk(2) raises when the marker
+ # dimension has width one.
+ top2 = _top_two(p)
feats = torch.stack([top2[:, 0], top2[:, 0] - top2[:, 1], ent, k / 255.0], -1)
pooled = h[:, 0].float()
act_logits = m.act_head(torch.cat([pooled, feats], -1))
diff --git a/laya/integrations/langchain.py b/laya/integrations/langchain.py
index 4c85a2f6..f7e5cb9b 100644
--- a/laya/integrations/langchain.py
+++ b/laya/integrations/langchain.py
@@ -9,7 +9,9 @@ deployments (your own `laya-serve`) without requiring PyTorch on edge clients.
from __future__ import annotations
import json
+import threading
import urllib.error
+import urllib.parse
import urllib.request
from typing import Any, Callable, Dict, Optional, Sequence, Union
@@ -77,6 +79,23 @@ def _extract_from_messages_list(msgs: Sequence[Any]) -> str:
return str(getattr(last, "content", last))
+class _SameOriginRedirectHandler(urllib.request.HTTPRedirectHandler):
+ """Do not forward bearer credentials across an origin or HTTPS downgrade."""
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ old = urllib.parse.urlsplit(req.full_url)
+ new = urllib.parse.urlsplit(newurl)
+ old_port = old.port or (443 if old.scheme.lower() == "https" else 80)
+ new_port = new.port or (443 if new.scheme.lower() == "https" else 80)
+ if (
+ old.scheme.lower() != new.scheme.lower()
+ or (old.hostname or "").lower() != (new.hostname or "").lower()
+ or old_port != new_port
+ ):
+ raise urllib.error.URLError("refusing cross-origin or HTTPS-downgrade redirect")
+ return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
def _call_remote(
base_url: str,
state: Any,
@@ -100,8 +119,9 @@ def _call_remote(
headers["Authorization"] = f"Bearer {api_key}"
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
+ opener = urllib.request.build_opener(_SameOriginRedirectHandler())
try:
- with urllib.request.urlopen(req, timeout=timeout) as resp:
+ with opener.open(req, timeout=timeout) as resp:
return json.loads(resp.read().decode("utf-8"))
except urllib.error.HTTPError as e:
body = e.read().decode("utf-8", errors="replace")
@@ -111,13 +131,16 @@ def _call_remote(
_DEFAULT_ROUTER = None
+_DEFAULT_ROUTER_LOCK = threading.Lock()
def _get_default_router():
global _DEFAULT_ROUTER
if _DEFAULT_ROUTER is None:
- from ..router import Router
- _DEFAULT_ROUTER = Router()
+ with _DEFAULT_ROUTER_LOCK:
+ if _DEFAULT_ROUTER is None:
+ from ..router import Router
+ _DEFAULT_ROUTER = Router()
return _DEFAULT_ROUTER
diff --git a/laya/mcp/server.py b/laya/mcp/server.py
index b3b67602..eea0847a 100644
--- a/laya/mcp/server.py
+++ b/laya/mcp/server.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import json
import os
import sys
+import threading
from importlib import metadata as _metadata
from typing import Any
@@ -54,6 +55,7 @@ except Exception: # running from a source checkout without install metadata
server = MCPServer("laya", version=_LAYA_VERSION)
_ROUTER: Any = None
+_ROUTER_LOCK = threading.Lock()
# MCP default preload list. laya.serve preloads every checkpoint when LAYA_MODELS
# is empty; MCP keeps typed-decisions lazy on purpose (it is ~as big as the other
@@ -95,19 +97,22 @@ def _ensure_router() -> Any:
global _ROUTER
if _ROUTER is not None:
return _ROUTER
- try:
- from laya import Router
- except Exception as exc:
- raise ToolError("internal_error", f"cannot import laya: {exc}") from exc
- try:
- _apply_thread_limit()
- router = Router(device=env_device())
- if _env_bool("LAYA_PRELOAD", True):
- router.preload(_models_from_env())
- except Exception as exc:
- raise ToolError("internal_error", f"router construction failed: {exc}") from exc
- _ROUTER = router
- return router
+ with _ROUTER_LOCK:
+ if _ROUTER is not None:
+ return _ROUTER
+ try:
+ from laya import Router
+ except Exception as exc:
+ raise ToolError("internal_error", f"cannot import laya: {exc}") from exc
+ try:
+ _apply_thread_limit()
+ router = Router(device=env_device())
+ if _env_bool("LAYA_PRELOAD", True):
+ router.preload(_models_from_env())
+ except Exception as exc:
+ raise ToolError("internal_error", f"router construction failed: {exc}") from exc
+ _ROUTER = router
+ return router
def _dump(payload: Any) -> str:
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 58e3ff2c..b7f2e2d6 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -100,7 +100,11 @@ class ONNXAgent(HookRegistry):
f"ONNX model not found at {onnx_path!r}. Please run export_onnx.py first."
)
- # Load Tokenizer
+ # Load Tokenizer. Keep this compatibility fix in sync with Agent: checkpoints
+ # produced by newer Transformers versions can contain TokenizersBackend or a list-valued
+ # extra_special_tokens field that older loaders cannot parse.
+ from .agent import _fix_tokenizer_config
+ _fix_tokenizer_config(model_dir)
tok_dir = os.path.join(model_dir, "tokenizer")
self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))
@@ -111,7 +115,19 @@ class ONNXAgent(HookRegistry):
if "CUDAExecutionProvider" in available
else ["CPUExecutionProvider"]
)
- self.session = ort.InferenceSession(onnx_path, providers=providers)
+ try:
+ self.session = ort.InferenceSession(onnx_path, providers=providers)
+ except Exception:
+ # A provider can be listed by onnxruntime yet still fail to initialize
+ # (missing CUDA libraries, unsupported driver, mismatched DLLs). Keep
+ # the CPU runtime usable instead of making model construction fail.
+ if providers == ["CPUExecutionProvider"]:
+ raise
+ warnings.warn(
+ "laya ONNX: CUDAExecutionProvider initialization failed; falling back to CPUExecutionProvider.",
+ RuntimeWarning, stacklevel=2,
+ )
+ self.session = ort.InferenceSession(onnx_path, providers=["CPUExecutionProvider"])
self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])
self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})
@@ -198,6 +214,12 @@ class ONNXAgent(HookRegistry):
from .agent import Agent as _Agent
ids = list(questions.keys())
+ if not ids:
+ return {
+ "model": "laya-rl-agent-onnx",
+ "answers": {},
+ "usage": {"input_tokens": 0, "output_tokens": 0},
+ }
for qid in ids:
_Agent._check_question(qid, questions[qid])
items = []
@@ -206,7 +228,10 @@ class ONNXAgent(HookRegistry):
for qid in ids:
q = self._to_internal(questions[qid])
- seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len)
+ seq, markers = build_sequence(
+ self.tok, state, q, max_len, head_max_len,
+ truncate_left=isinstance(state, list),
+ )
if len(markers) != len(render_options(q)):
raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})
diff --git a/laya/serve.py b/laya/serve.py
index 94e3a660..c0fc1bc3 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -35,6 +35,7 @@ and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.
import hmac
import json
import os
+from contextlib import asynccontextmanager
from typing import Any, Dict, Optional
# The three checkpoint names the router understands; used to decide whether a
@@ -197,9 +198,19 @@ def create_app(router: Optional[Any] = None):
# exists (module scope, TestClient startup, a preload script).
gate: Optional[asyncio.Lock] = None
+ @asynccontextmanager
+ async def lifespan(_app: FastAPI):
+ try:
+ yield
+ finally:
+ # TestClient, embedded ASGI apps, and process supervisors all need
+ # the executor to drain when the app stops.
+ pool.shutdown(wait=True, cancel_futures=True)
+
app = FastAPI(
title="laya-serve",
summary="Laya System-1 decisions over the TypeSafe Jev /v1/systemone protocol",
+ lifespan=lifespan,
)
# Compared as bytes, not str. `hmac.compare_digest` raises TypeError when a str
diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py
new file mode 100644
index 00000000..3c90c0b3
--- /dev/null
+++ b/tests/test_audit_regressions.py
@@ -0,0 +1,210 @@
+"""Regression tests for fixes found during the cross-runtime audit."""
+import io
+import json
+from contextlib import redirect_stdout
+
+from laya.cli import show_answers
+from laya.integrations.langchain import _SameOriginRedirectHandler
+from laya.onnx_agent import ONNXAgent
+from laya.serve import MAX_BODY_BYTES, create_app
+
+
+class FakeRouter:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None):
+ return {
+ "model": "test",
+ "answers": {},
+ "usage": {"input_tokens": 0, "output_tokens": 0},
+ }
+
+
+def test_serve_rejects_oversized_stream_without_content_length():
+ from fastapi.testclient import TestClient
+
+ app = create_app(router=FakeRouter())
+ client = TestClient(app)
+ payload = {
+ "state": "hello",
+ "questions": {"q": {"type": "noul", "instructions": "?"}},
+ "padding": "x" * (MAX_BODY_BYTES + 1024),
+ }
+ encoded = json.dumps(payload).encode()
+ # An iterator makes httpx use a streaming body, so this exercises the
+ # Content-Length-independent path rather than the existing header guard.
+ response = client.post(
+ "/v1/systemone",
+ content=iter([encoded[:32], encoded[32:]]),
+ headers={"content-type": "application/json"},
+ )
+ assert response.status_code == 413
+
+
+def test_serve_lifespan_shuts_down_inference_executor():
+ from fastapi.testclient import TestClient
+ from unittest.mock import patch
+
+ with patch("concurrent.futures.ThreadPoolExecutor.shutdown") as shutdown:
+ with TestClient(create_app(router=FakeRouter())):
+ pass
+ shutdown.assert_called_once_with(wait=True, cancel_futures=True)
+
+
+def test_serve_non_ascii_auth_is_401_not_500(monkeypatch):
+ from fastapi.testclient import TestClient
+
+ monkeypatch.setenv("LAYA_API_KEY", "s3cret")
+ app = create_app(router=FakeRouter())
+ client = TestClient(app)
+ response = client.post(
+ "/v1/systemone",
+ content=b'{"state":"hello","questions":{}}',
+ headers=[
+ ("content-type", "application/json"),
+ (b"authorization", b"Bearer caf\xe9"),
+ ],
+ )
+ assert response.status_code == 401
+
+
+def test_onnx_empty_questions_matches_agent_contract():
+ agent = ONNXAgent.__new__(ONNXAgent)
+ result = ONNXAgent._infer(agent, "hello", {})
+ assert result == {
+ "model": "laya-rl-agent-onnx",
+ "answers": {},
+ "usage": {"input_tokens": 0, "output_tokens": 0},
+ }
+
+
+def test_onnx_long_list_state_keeps_newest_turn(monkeypatch):
+ import laya.onnx_agent as module
+
+ seen = []
+ original = module.build_sequence
+
+ def fake_build(*args, **kwargs):
+ seen.append(kwargs.get("truncate_left"))
+ raise RuntimeError("stop after encoding")
+
+ monkeypatch.setattr(module, "build_sequence", fake_build)
+ agent = ONNXAgent.__new__(ONNXAgent)
+ agent.cfg = {"max_len": 32, "head_max_len": 16}
+ agent.temperature = [1.0, 1.0, 1.0]
+ agent.temperature_by_options = {}
+ agent.tok = object()
+ try:
+ ONNXAgent._infer(agent, [{"text": "old"}, {"text": "new"}], {
+ "q": {"type": "noul", "instructions": "?"}
+ })
+ except RuntimeError as exc:
+ assert str(exc) == "stop after encoding"
+ else:
+ raise AssertionError("encoding should have been reached")
+ assert seen == [True]
+
+
+def test_cli_renders_choice_probability_from_probabilities_map():
+ output = io.StringIO()
+ with redirect_stdout(output):
+ show_answers({
+ "answers": {
+ "q": {"type": "choice", "choice": "yes", "probabilities": {"yes": 0.91, "no": 0.09}}
+ }
+ })
+ assert "yes (p=0.910)" in output.getvalue()
+
+
+def test_default_langchain_router_initialization_is_singleton(monkeypatch):
+ import threading
+
+ import laya.integrations.langchain as module
+ import laya.router as router_module
+
+ created = []
+ lock = threading.Lock()
+
+ class FakeRouter:
+ def __init__(self):
+ with lock:
+ created.append(self)
+
+ monkeypatch.setattr(router_module, "Router", FakeRouter)
+ monkeypatch.setattr(module, "_DEFAULT_ROUTER", None)
+ barrier = threading.Barrier(2)
+ results = []
+
+ def get_router():
+ barrier.wait()
+ results.append(module._get_default_router())
+
+ threads = [threading.Thread(target=get_router) for _ in range(2)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ assert len(created) == 1
+ assert results[0] is results[1]
+
+
+def test_remote_redirect_handler_rejects_cross_origin_credentials():
+ import urllib.error
+ import urllib.request
+
+ request = urllib.request.Request(
+ "https://laya.example/v1/systemone",
+ headers={"Authorization": "Bearer secret"},
+ )
+ handler = _SameOriginRedirectHandler()
+ same_origin = handler.redirect_request(
+ request, None, 302, "Found", {}, "https://laya.example/v1/systemone?retry=1"
+ )
+ assert same_origin.get_header("Authorization") == "Bearer secret"
+ try:
+ handler.redirect_request(
+ request, None, 302, "Found", {}, "http://attacker.example/collect"
+ )
+ except urllib.error.URLError as exc:
+ assert "cross-origin" in str(exc)
+ else:
+ raise AssertionError("cross-origin redirect was accepted")
+
+
+def test_fast_path_rejects_sequences_over_its_capacity():
+ from types import SimpleNamespace
+
+ import torch
+
+ from laya.agent import Agent
+
+ agent = Agent.__new__(Agent)
+ agent._fast = SimpleNamespace(max_len=16)
+ agent.device = torch.device("cpu")
+ agent.dtype = torch.float32
+ agent.amp_enabled = False
+ agent.mps_amp_min_rows = 5
+ batch = {
+ "input_ids": torch.zeros((1, 17), dtype=torch.long),
+ "attention_mask": torch.ones((1, 17), dtype=torch.long),
+ "marker_pos": torch.zeros((1, 2), dtype=torch.long),
+ "marker_mask": torch.ones((1, 2), dtype=torch.bool),
+ "qtype": torch.zeros((1,), dtype=torch.long),
+ }
+ try:
+ agent._infer(batch)
+ except ValueError as exc:
+ assert "max_len=16" in str(exc)
+ else:
+ raise AssertionError("oversized fast-path request was accepted")
+
+
+def test_fast_top_two_handles_one_option():
+ import pytest
+ pytest.importorskip("tilelang")
+ from laya.fast import _top_two
+ import torch
+
+ values = _top_two(torch.tensor([[0.75]]))
+ assert values.shape == (1, 2)
+ assert torch.allclose(values, torch.tensor([[0.75, 0.0]]))