Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(ts): tokenize the state once per systemOne call, not once per question (#309)
Aashish Mahato609de1befd09
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex ab16bc2a..4e732b38 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -1,11 +1,13 @@import {TEMP_MAX,TEMP_MIN,- buildSequence,+ buildQuestionPrefix,clampTemperature,collateItems,confidenceFromProbs,renderOptions,+ sequenceWithState,+ serializeState,softmax,tempBucket,} from "./common.js";@@ -249,11 +251,15 @@ export class Agent {}const items: { ids: number[]; markers: number[]; qtype: number }[] = [];const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[] = [];+ // The state text is shared by every question and is usually the longest text in the+ // sequence — encode it once and compose the per-question prefixes onto it.+ const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));for (const qid of ids) {checkQuestion(qid, questions[qid]);const q = toInternal(questions[qid]);internals.push(q);- const { ids: seq, markers } = buildSequence(this.tok, state, q, this.maxLen, this.headMaxLen);+ const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen);+ const { ids: seq, markers } = sequenceWithState(prefix, stAll, this.tok.sepId, this.maxLen);if (markers.length !== renderOptions(q).length) {throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`);}diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex cf366b98..014a0212 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -42,8 +42,17 @@ export function renderOptions(q: InternalQ): string[] {"true: " + (t !== null && t !== undefined && t !== "" ? renderCriterion(t) : "yes, the statement holds"),];}-export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,- maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[] } {+export interface QuestionPrefix {+ /** [CLS] head [SEP] options [SEP] — the state-independent part of the sequence. */+ ids: number[];+ /** Mask-marker positions (absolute; the prefix sits at the start of the final sequence). */+ markers: number[];+ nOptions: number;+}+/** The question half of `buildSequence`: everything before the state tokens. Hoisted out so+ * callers asking several questions about the same state can encode the state text only once. */+export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,+ maxLen = 512, headMaxLen = 192, optionOrder?: number[]): QuestionPrefix {const maskTok = tok.maskToken;const opts = renderOptions(q);const order = optionOrder ?? opts.map((_, i) => i);@@ -58,15 +67,26 @@ export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,budget = headMaxLen - optIds.reduce((a, o) => a + o.length, 0);}headIds = headIds.slice(0, Math.max(8, budget));- let ids = [tok.clsId, ...headIds, tok.sepId];+ const ids = [tok.clsId, ...headIds, tok.sepId];const markers: number[] = [];for (const o of optIds) { markers.push(ids.length); ids.push(...o); }ids.push(tok.sepId);- const room = Math.max(0, maxLen - ids.length - 1);- const stAll = tok.encode(serializeState(state).split(maskTok).join(" "));- const st = truncateLeft ? stAll.slice(-room) : stAll.slice(0, room);- ids = [...ids, ...st, tok.sepId].slice(0, maxLen);- return { ids, markers: markers.filter((m) => m < maxLen) };+ return { ids, markers, nOptions: opts.length };+}+/** Append pre-encoded state tokens to a question prefix. Identical output to building the+ * whole sequence in one pass, but the state only needs encoding once per state, not once+ * per (state, question) pair. */+export function sequenceWithState(prefix: QuestionPrefix, stateIds: number[], sepId: number,+ maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[] } {+ const room = Math.max(0, maxLen - prefix.ids.length - 1);+ const st = truncateLeft ? stateIds.slice(-room) : stateIds.slice(0, room);+ const ids = [...prefix.ids, ...st, sepId].slice(0, maxLen);+ return { ids, markers: prefix.markers.filter((m) => m < maxLen) };+}+export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,+ maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[] } {+ const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));+ return sequenceWithState(buildQuestionPrefix(tok, q, maxLen, headMaxLen, optionOrder), stAll, tok.sepId, maxLen, truncateLeft);}export function softmax(z: number[]): number[] {const m = Math.max(...z);diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex 178cb90a..7496e360 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -26,6 +26,8 @@ export {renderOptions,serializeState,buildSequence,+ buildQuestionPrefix,+ sequenceWithState,softmax,confidenceFromProbs,clampTemperature,@@ -34,6 +36,6 @@ export {TEMP_MIN,TEMP_MAX,} from "./common.js";-export type { QType, InternalQ, CollateItem, CollatedBatch } from "./common.js";+export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix } from "./common.js";export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js";diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.tsindex e4faf79c..24cfe8f7 100644--- a/laya-ts/src/tokenizer.ts+++ b/laya-ts/src/tokenizer.ts@@ -73,11 +73,12 @@ function bpeWord(chars: string[], rank: Map<string, number>): string[] {}/** Byte-level BPE encode: NFC-normalize, NO lowercasing, GPT-2 byte map + rank-order merges. */+let sharedEncoder: TextEncoder | null = null;export function bpeEncode(vocab: Map<string, number>, merges: Map<string, number>, text: string): number[] {const { b2u } = maps();const unkId = vocab.get("[UNK]") ?? CHECKPOINT_IDS.unk;const out: number[] = [];- const enc = new TextEncoder();+ const enc = (sharedEncoder ??= new TextEncoder());const parts = text.normalize("NFC").match(GPT2_SPLIT);if (!parts) return out;for (const piece of parts) {diff --git a/laya-ts/tests/sequence-prefix.test.ts b/laya-ts/tests/sequence-prefix.test.tsnew file mode 100644index 00000000..fe7bd2fe--- /dev/null+++ b/laya-ts/tests/sequence-prefix.test.ts@@ -0,0 +1,98 @@+// laya-ts/tests/sequence-prefix.test.ts — buildQuestionPrefix/sequenceWithState must compose+// into exactly what buildSequence produced as one pass, and Agent must encode the state once.+import { describe, expect, it } from "vitest";+import { Agent, defaultTokenizer } from "../src/agent.js";+import {+ buildQuestionPrefix,+ buildSequence,+ renderOptions,+ sequenceWithState,+ serializeState,+ type InternalQ,+} from "../src/common.js";++const QUESTIONS: Array<[string, InternalQ]> = [+ ["choice", { t: "choice", ins: "What does the customer want?", crit: { refund: "money back", apology: "", other: "rest" } }],+ ["score", { t: "score", ins: "Rate satisfaction", crit: ["terrible", "ok", "great"] }],+ ["noul-empty", { t: "noul", ins: "The statement holds", crit: {} }],+ ["noul-custom", { t: "noul", ins: "Angry?", crit: { true: "yes, furious", false: "no" } }],+];++const LONG = "charged twice on my card and nobody replied to my tickets ".repeat(40);++const STATES: Array<[string, unknown]> = [+ ["short", "refund please"],+ ["long", LONG],+ ["object", { body: "charged twice", meta: { order: 88121, tags: ["billing", "urgent"] } }],+ ["mask-token", "why is [MASK] in my email [MASK] body?"],+];++const GEOMETRIES: Array<[string, number, number, number[] | undefined, boolean]> = [+ ["default", 512, 192, undefined, false],+ ["state-truncated", 64, 192, undefined, false],+ ["truncate-left", 64, 192, undefined, true],+ ["tight-head", 128, 40, undefined, false],+ ["prefix-overflow", 48, 192, undefined, false],+ ["reordered", 512, 192, [2, 0, 1], false],+];++describe("question prefix composition", () => {+ for (const [qn, q] of QUESTIONS) {+ for (const [sn, state] of STATES) {+ for (const [gn, maxLen, headMaxLen, order, left] of GEOMETRIES) {+ if (order && q.t !== "choice") continue; // optionOrder only applies to choice here+ it(`${qn} / ${sn} / ${gn} matches one-pass buildSequence`, () => {+ const tok = defaultTokenizer();+ const expected = buildSequence(tok, state, q, maxLen, headMaxLen, order, left);+ const prefix = buildQuestionPrefix(tok, q, maxLen, headMaxLen, order);+ const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));+ const actual = sequenceWithState(prefix, stAll, tok.sepId, maxLen, left);+ expect(actual).toEqual(expected);+ expect(prefix.nOptions).toBe(renderOptions(q).length);+ });+ }+ }+ }+});++describe("Agent state tokenization", () => {+ it("encodes the state text exactly once per systemOne call", async () => {+ const base = defaultTokenizer();+ const stateTexts: string[] = [];+ let encodeCalls = 0;+ const tok = {+ ...base,+ encode(text: string): number[] {+ encodeCalls++;+ stateTexts.push(text);+ return base.encode(text);+ },+ };+ const provider = {+ async runEncoder(batch: { inputIds: number[][] }) {+ return { lastHidden: batch.inputIds.map((row) => row.map(() => [1, 0])) };+ },+ async runHead(_h: unknown, batch: { markerPos: number[][] }) {+ return {+ logits: batch.markerPos.map((m) => m.map((_, i) => i + 1)),+ act: batch.markerPos.map(() => [3, 0]),+ };+ },+ };+ const agent = new Agent({ provider, tok } as never);+ const questions = {+ intent: { type: "choice", instructions: "?", criteria: { refund: "r", apology: "a", other: "o" } },+ angry: { type: "noul", instructions: "angry?" },+ csat: { type: "score", instructions: "rate", criteria: ["bad", "ok", "great"] },+ } as never;+ const state = { body: "charged twice, refund please" };+ const r = await agent.systemOne(state, questions);++ const serialized = serializeState(state);+ const stateEncodes = stateTexts.filter((t) => t === serialized).length;+ expect(stateEncodes).toBe(1); // not once per question+ // 1 state + per question (1 head + K options): 3 + 2 + 3 options/head sums = 1 + 4 + 3 + 4+ expect(encodeCalls).toBe(1 + (1 + 3) + (1 + 2) + (1 + 3));+ expect(Object.keys(r.answers).sort()).toEqual(["angry", "csat", "intent"]);+ });+});