Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(ts): prefix cache and hooks fast-path

nnlgsakib76ae05e4b357
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index 6e4ec30e..7a3df97b 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -67,8 +67,25 @@ export interface QuestionPrefix {
}
/** The question half of `buildSequence`: everything before the state tokens. Hoisted out so
* callers asking several questions about the same state can encode the state text only once. */
+// Per-tokenizer prefix cache (cap 1k, clear on overflow); repeat triage skips re-encode.
+const prefixCache = new WeakMap<object, Map<string, QuestionPrefix>>();
export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,
maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix {
+ let per = prefixCache.get(tok as object);
+ if (!per) {
+ per = new Map();
+ prefixCache.set(tok as object, per);
+ }
+ const key = JSON.stringify([q.t, q.ins, q.crit, q.labels ?? null, maxLen, headMaxLen, optionOrder ?? null]);
+ const hit = per.get(key);
+ if (hit) return hit;
+ const built = buildQuestionPrefixUncached(tok, q, maxLen, headMaxLen, optionOrder);
+ if (per.size > 1000) per.clear();
+ per.set(key, built);
+ return built;
+}
+function buildQuestionPrefixUncached(tok: TokenizerLike, q: InternalQ,
+ maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix {
const maskTok = tok.maskToken;
const opts = renderOptions(q);
const order = optionOrder ?? opts.map((_, i) => i);
diff --git a/laya-ts/src/hooks.ts b/laya-ts/src/hooks.ts
index 8d5428ac..cbd198a2 100644
--- a/laya-ts/src/hooks.ts
+++ b/laya-ts/src/hooks.ts
@@ -231,9 +231,18 @@ export function composeHooks(
onPredictStart?: PredictHookArg,
onPredictEnd?: PredictHookArg,
): Hook[] {
+ if (
+ defaultHooksList.length === 0 && installed.length === 0 && (hooks === null || hooks === undefined) &&
+ (onPredictStart === null || onPredictStart === undefined) &&
+ (onPredictEnd === null || onPredictEnd === undefined)
+ ) {
+ return EMPTY_HOOKS;
+ }
return [...defaultHooksList, ...installed, ...normaliseHooks(hooks, onPredictStart, onPredictEnd)];
}
+const EMPTY_HOOKS: Hook[] = [];
+
/**
* Base class giving a runtime-mutable hook list.
*
diff --git a/laya-ts/tests/prefix-cache.test.ts b/laya-ts/tests/prefix-cache.test.ts
new file mode 100644
index 00000000..503fa475
--- /dev/null
+++ b/laya-ts/tests/prefix-cache.test.ts
@@ -0,0 +1,14 @@
+import { describe, expect, it } from "vitest";
+import { buildQuestionPrefix } from "../src/common.js";
+import { defaultTokenizer } from "../src/agent.js";
+
+describe("prefix cache", () => {
+ it("returns cached prefix on repeat", () => {
+ const tok = defaultTokenizer();
+ const q: any = { t: "choice", ins: "What?", crit: { a: "x", b: "y" } };
+ const p1 = buildQuestionPrefix(tok, q, 512, 192);
+ const p2 = buildQuestionPrefix(tok, q, 512, 192);
+ expect(p2).toBe(p1);
+ expect(p2.ids).toEqual(p1.ids);
+ });
+});