Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(ts): tokenize the state once per systemOne call, not once per question (#309)

Aashish Mahato609de1befd09
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index ab16bc2a..4e732b38 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -1,11 +1,13 @@
import {
TEMP_MAX,
TEMP_MIN,
- buildSequence,
+ buildQuestionPrefix,
clampTemperature,
collateItems,
confidenceFromProbs,
renderOptions,
+ sequenceWithState,
+ serializeState,
softmax,
tempBucket,
} from "./common.js";
@@ -249,11 +251,15 @@ export class Agent {
}
const items: { ids: number[]; markers: number[]; qtype: number }[] = [];
const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[] = [];
+ // The state text is shared by every question and is usually the longest text in the
+ // sequence — encode it once and compose the per-question prefixes onto it.
+ const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));
for (const qid of ids) {
checkQuestion(qid, questions[qid]);
const q = toInternal(questions[qid]);
internals.push(q);
- const { ids: seq, markers } = buildSequence(this.tok, state, q, this.maxLen, this.headMaxLen);
+ const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen);
+ const { ids: seq, markers } = sequenceWithState(prefix, stAll, this.tok.sepId, this.maxLen);
if (markers.length !== renderOptions(q).length) {
throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`);
}
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index cf366b98..014a0212 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -42,8 +42,17 @@ export function renderOptions(q: InternalQ): string[] {
"true: " + (t !== null && t !== undefined && t !== "" ? renderCriterion(t) : "yes, the statement holds"),
];
}
-export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,
- maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[] } {
+export interface QuestionPrefix {
+ /** [CLS] head [SEP] options [SEP] — the state-independent part of the sequence. */
+ ids: number[];
+ /** Mask-marker positions (absolute; the prefix sits at the start of the final sequence). */
+ markers: number[];
+ nOptions: number;
+}
+/** The question half of `buildSequence`: everything before the state tokens. Hoisted out so
+ * callers asking several questions about the same state can encode the state text only once. */
+export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,
+ maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix {
const maskTok = tok.maskToken;
const opts = renderOptions(q);
const order = optionOrder ?? opts.map((_, i) => i);
@@ -58,15 +67,26 @@ export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,
budget = headMaxLen - optIds.reduce((a, o) => a + o.length, 0);
}
headIds = headIds.slice(0, Math.max(8, budget));
- let ids = [tok.clsId, ...headIds, tok.sepId];
+ const ids = [tok.clsId, ...headIds, tok.sepId];
const markers: number[] = [];
for (const o of optIds) { markers.push(ids.length); ids.push(...o); }
ids.push(tok.sepId);
- const room = Math.max(0, maxLen - ids.length - 1);
- const stAll = tok.encode(serializeState(state).split(maskTok).join(" "));
- const st = truncateLeft ? stAll.slice(-room) : stAll.slice(0, room);
- ids = [...ids, ...st, tok.sepId].slice(0, maxLen);
- return { ids, markers: markers.filter((m) => m < maxLen) };
+ return { ids, markers, nOptions: opts.length };
+}
+/** Append pre-encoded state tokens to a question prefix. Identical output to building the
+ * whole sequence in one pass, but the state only needs encoding once per state, not once
+ * per (state, question) pair. */
+export function sequenceWithState(prefix: QuestionPrefix, stateIds: number[], sepId: number,
+ maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[] } {
+ const room = Math.max(0, maxLen - prefix.ids.length - 1);
+ const st = truncateLeft ? stateIds.slice(-room) : stateIds.slice(0, room);
+ const ids = [...prefix.ids, ...st, sepId].slice(0, maxLen);
+ return { ids, markers: prefix.markers.filter((m) => m < maxLen) };
+}
+export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,
+ maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[] } {
+ const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));
+ return sequenceWithState(buildQuestionPrefix(tok, q, maxLen, headMaxLen, optionOrder), stAll, tok.sepId, maxLen, truncateLeft);
}
export function softmax(z: number[]): number[] {
const m = Math.max(...z);
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index 178cb90a..7496e360 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -26,6 +26,8 @@ export {
renderOptions,
serializeState,
buildSequence,
+ buildQuestionPrefix,
+ sequenceWithState,
softmax,
confidenceFromProbs,
clampTemperature,
@@ -34,6 +36,6 @@ export {
TEMP_MIN,
TEMP_MAX,
} from "./common.js";
-export type { QType, InternalQ, CollateItem, CollatedBatch } from "./common.js";
+export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix } from "./common.js";
export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";
export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js";
diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.ts
index e4faf79c..24cfe8f7 100644
--- a/laya-ts/src/tokenizer.ts
+++ b/laya-ts/src/tokenizer.ts
@@ -73,11 +73,12 @@ function bpeWord(chars: string[], rank: Map<string, number>): string[] {
}
/** Byte-level BPE encode: NFC-normalize, NO lowercasing, GPT-2 byte map + rank-order merges. */
+let sharedEncoder: TextEncoder | null = null;
export function bpeEncode(vocab: Map<string, number>, merges: Map<string, number>, text: string): number[] {
const { b2u } = maps();
const unkId = vocab.get("[UNK]") ?? CHECKPOINT_IDS.unk;
const out: number[] = [];
- const enc = new TextEncoder();
+ const enc = (sharedEncoder ??= new TextEncoder());
const parts = text.normalize("NFC").match(GPT2_SPLIT);
if (!parts) return out;
for (const piece of parts) {
diff --git a/laya-ts/tests/sequence-prefix.test.ts b/laya-ts/tests/sequence-prefix.test.ts
new file mode 100644
index 00000000..fe7bd2fe
--- /dev/null
+++ b/laya-ts/tests/sequence-prefix.test.ts
@@ -0,0 +1,98 @@
+// laya-ts/tests/sequence-prefix.test.ts — buildQuestionPrefix/sequenceWithState must compose
+// into exactly what buildSequence produced as one pass, and Agent must encode the state once.
+import { describe, expect, it } from "vitest";
+import { Agent, defaultTokenizer } from "../src/agent.js";
+import {
+ buildQuestionPrefix,
+ buildSequence,
+ renderOptions,
+ sequenceWithState,
+ serializeState,
+ type InternalQ,
+} from "../src/common.js";
+
+const QUESTIONS: Array<[string, InternalQ]> = [
+ ["choice", { t: "choice", ins: "What does the customer want?", crit: { refund: "money back", apology: "", other: "rest" } }],
+ ["score", { t: "score", ins: "Rate satisfaction", crit: ["terrible", "ok", "great"] }],
+ ["noul-empty", { t: "noul", ins: "The statement holds", crit: {} }],
+ ["noul-custom", { t: "noul", ins: "Angry?", crit: { true: "yes, furious", false: "no" } }],
+];
+
+const LONG = "charged twice on my card and nobody replied to my tickets ".repeat(40);
+
+const STATES: Array<[string, unknown]> = [
+ ["short", "refund please"],
+ ["long", LONG],
+ ["object", { body: "charged twice", meta: { order: 88121, tags: ["billing", "urgent"] } }],
+ ["mask-token", "why is [MASK] in my email [MASK] body?"],
+];
+
+const GEOMETRIES: Array<[string, number, number, number[] | undefined, boolean]> = [
+ ["default", 512, 192, undefined, false],
+ ["state-truncated", 64, 192, undefined, false],
+ ["truncate-left", 64, 192, undefined, true],
+ ["tight-head", 128, 40, undefined, false],
+ ["prefix-overflow", 48, 192, undefined, false],
+ ["reordered", 512, 192, [2, 0, 1], false],
+];
+
+describe("question prefix composition", () => {
+ for (const [qn, q] of QUESTIONS) {
+ for (const [sn, state] of STATES) {
+ for (const [gn, maxLen, headMaxLen, order, left] of GEOMETRIES) {
+ if (order && q.t !== "choice") continue; // optionOrder only applies to choice here
+ it(`${qn} / ${sn} / ${gn} matches one-pass buildSequence`, () => {
+ const tok = defaultTokenizer();
+ const expected = buildSequence(tok, state, q, maxLen, headMaxLen, order, left);
+ const prefix = buildQuestionPrefix(tok, q, maxLen, headMaxLen, order);
+ const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));
+ const actual = sequenceWithState(prefix, stAll, tok.sepId, maxLen, left);
+ expect(actual).toEqual(expected);
+ expect(prefix.nOptions).toBe(renderOptions(q).length);
+ });
+ }
+ }
+ }
+});
+
+describe("Agent state tokenization", () => {
+ it("encodes the state text exactly once per systemOne call", async () => {
+ const base = defaultTokenizer();
+ const stateTexts: string[] = [];
+ let encodeCalls = 0;
+ const tok = {
+ ...base,
+ encode(text: string): number[] {
+ encodeCalls++;
+ stateTexts.push(text);
+ return base.encode(text);
+ },
+ };
+ const provider = {
+ async runEncoder(batch: { inputIds: number[][] }) {
+ return { lastHidden: batch.inputIds.map((row) => row.map(() => [1, 0])) };
+ },
+ async runHead(_h: unknown, batch: { markerPos: number[][] }) {
+ return {
+ logits: batch.markerPos.map((m) => m.map((_, i) => i + 1)),
+ act: batch.markerPos.map(() => [3, 0]),
+ };
+ },
+ };
+ const agent = new Agent({ provider, tok } as never);
+ const questions = {
+ intent: { type: "choice", instructions: "?", criteria: { refund: "r", apology: "a", other: "o" } },
+ angry: { type: "noul", instructions: "angry?" },
+ csat: { type: "score", instructions: "rate", criteria: ["bad", "ok", "great"] },
+ } as never;
+ const state = { body: "charged twice, refund please" };
+ const r = await agent.systemOne(state, questions);
+
+ const serialized = serializeState(state);
+ const stateEncodes = stateTexts.filter((t) => t === serialized).length;
+ expect(stateEncodes).toBe(1); // not once per question
+ // 1 state + per question (1 head + K options): 3 + 2 + 3 options/head sums = 1 + 4 + 3 + 4
+ expect(encodeCalls).toBe(1 + (1 + 3) + (1 + 2) + (1 + 3));
+ expect(Object.keys(r.answers).sort()).toEqual(["angry", "csat", "intent"]);
+ });
+});