Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): report state truncation in usage from the real token budget
Aashisha4f4c41f1b15
diff --git a/laya-ts/README.md b/laya-ts/README.mdindex 5e4a6148..8d6e7c23 100644--- a/laya-ts/README.md+++ b/laya-ts/README.md@@ -98,6 +98,23 @@ instead of a schema to get raw answers. Zod/TypeBox users can pass `z.toJSONScheany object with a `toJSONSchema()` method is accepted. `planFromJsonSchema`,`questionsFromJsonSchema` and `answersToJson` expose the planning and projection steps.+## Truncation reporting++The state is clamped to whatever token room a question's head leaves, and that budget moves+with `max_len`, `head_max_len` and the rendered head — so `usage` reports it instead of letting+you guess from character counts (issue #174; mirrors Python #181):++```ts+const out = await agent.predict(longState, questions);+out.usage.truncated; // true when any state token was dropped+out.usage.state_tokens; // encoded length of the full state+out.usage.state_tokens_dropped; // worst case across the questions+out.usage.truncated_questions; // ids of the questions whose head left too little room+```++The fields are absent only where no state was ever encoded (empty question schema, or a+start hook that supplied the result).+## Batching (many states, one call)Port of `Agent.predict_batch` / `Router.route_batch` / `Router.predict_batch`. The throughputdiff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex d4635316..7f2e4e3b 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -13,6 +13,7 @@ import {softmax,tempBucket,} from "./common.js";+import type { SequenceStats } from "./common.js";import type { Batch, SessionProvider } from "./providers.js";import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";import { decide, type DecideOptions, type DecisionResult } from "./structured.js";@@ -73,6 +74,13 @@ export type SystemAnswer = ChoiceAnswer | ScoreAnswer | NoulAnswer;export interface SystemUsage {input_tokens: number;output_tokens: number;+ /** State-fit reporting (issue #174, mirrors Python #181). Present on inference results;+ * absent on hook-supplied (skipped) results and empty-question early returns. */+ state_tokens?: number;+ /** Worst case across the questions, which share one state but not one head budget. */+ state_tokens_dropped?: number;+ truncated?: boolean;+ truncated_questions?: string[];}export interface SystemOneResult {@@ -428,28 +436,30 @@ export class Agent extends HookRegistry {internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[],maxLen: number,headMaxLen: number,- ): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number } {+ ): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number; stats: SequenceStats[] } {// The state text is shared by every question and is usually the longest text in the// sequence — encode it once and compose the per-question prefixes onto it.const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));const items: { ids: number[]; markers: number[]; qtype: number }[] = [];+ const stats: SequenceStats[] = [];for (let qi = 0; qi < ids.length; qi++) {const qid = ids[qi];const q = internals[qi];const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen);- const { ids: seq, markers } = sequenceWithState(+ const seq = sequenceWithState(prefix,stAll,this.tok.sepId,maxLen,Array.isArray(state),);- if (markers.length !== renderOptions(q).length) {+ if (seq.markers.length !== renderOptions(q).length) {throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${headMaxLen}`);}- items.push({ ids: seq, markers, qtype: QTYPES[q.t] });+ items.push({ ids: seq.ids, markers: seq.markers, qtype: QTYPES[q.t] });+ stats.push(seq.stats);}- return { items, nTokens: items.reduce((a, it) => a + it.ids.length, 0) };+ return { items, nTokens: items.reduce((a, it) => a + it.ids.length, 0), stats };}/**@@ -526,10 +536,22 @@ export class Agent extends HookRegistry {);row++;}+ // How much of the state actually reached the encoder. Truncation is a token budget, so it+ // moves with maxLen, headMaxLen and each question's rendered head — a caller cannot infer+ // it from the length of the state it sent (issue #174, Python #181 parity).+ const st = built[s].stats;+ const dropped = st.reduce((a, x) => Math.max(a, x.state_tokens_dropped), 0);out.push({model: "laya-rl-agent",answers,- usage: { input_tokens: built[s].nTokens, output_tokens: 0 },+ usage: {+ input_tokens: built[s].nTokens,+ output_tokens: 0,+ state_tokens: st[0]?.state_tokens ?? 0,+ state_tokens_dropped: dropped,+ truncated: dropped > 0,+ truncated_questions: ids.filter((_, qi) => st[qi].truncated),+ },});}}diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.tsindex ae5b9da8..94fb9f01 100644--- a/laya-ts/src/common.ts+++ b/laya-ts/src/common.ts@@ -96,19 +96,45 @@ export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,ids.push(tok.sepId);return { ids, markers, nOptions: opts.length };}+export interface SequenceStats {+ /** Encoded length of the full state, before the window clamp. */+ state_tokens: number;+ /** State tokens that actually reached the encoder after the final maxLen clamp. */+ state_tokens_used: number;+ /** State tokens dropped by the clamp: state_tokens - state_tokens_used. */+ state_tokens_dropped: number;+ /** True when the clamp dropped any state token. */+ truncated: boolean;+}+/** Append pre-encoded state tokens to a question prefix. Identical output to building the* whole sequence in one pass, but the state only needs encoding once per state, not once- * per (state, question) pair. */+ * per (state, question) pair.+ *+ * The state is clamped to whatever room the head leaves; `stats` reports that clamp so callers+ * never have to guess it from the character length of what they sent (issue #174, Python #181). */export function sequenceWithState(prefix: QuestionPrefix, stateIds: number[], sepId: number,- maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[] } {+ maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[]; stats: SequenceStats } {const room = Math.max(0, maxLen - prefix.ids.length - 1);// not stateIds.slice(-room): with no room left, slice(-0) is the whole state rather than none of it- const st = truncateLeft ? stateIds.slice(Math.max(0, stateIds.length - room)) : stateIds.slice(0, room);- const ids = [...prefix.ids, ...st, sepId].slice(0, maxLen);- return { ids, markers: prefix.markers.filter((m) => m < maxLen) };+ const kept = truncateLeft ? stateIds.slice(Math.max(0, stateIds.length - room)) : stateIds.slice(0, room);+ const ids = [...prefix.ids, ...kept, sepId].slice(0, maxLen);+ // Count against the final clamp rather than `kept`: the clamp is what actually decided+ // which state tokens reached the encoder.+ const used = Math.max(0, Math.min(kept.length, maxLen - prefix.ids.length));+ return {+ ids,+ markers: prefix.markers.filter((m) => m < maxLen),+ stats: {+ state_tokens: stateIds.length,+ state_tokens_used: used,+ state_tokens_dropped: stateIds.length - used,+ truncated: used < stateIds.length,+ },+ };}export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,- maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[] } {+ maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[]; stats: SequenceStats } {const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));return sequenceWithState(buildQuestionPrefix(tok, q, maxLen, headMaxLen, optionOrder), stAll, tok.sepId, maxLen, truncateLeft);}diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex b311d389..25896c67 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -64,6 +64,6 @@ export {TEMP_MIN,TEMP_MAX,} from "./common.js";-export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix } from "./common.js";+export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix, SequenceStats } from "./common.js";export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js";diff --git a/laya-ts/tests/truncation.test.ts b/laya-ts/tests/truncation.test.tsnew file mode 100644index 00000000..4e3aa077--- /dev/null+++ b/laya-ts/tests/truncation.test.ts@@ -0,0 +1,166 @@+import { describe, expect, it } from "vitest";+import { Agent } from "../src/agent.js";+import { buildSequence } from "../src/common.js";+import type { TokenizerLike } from "../src/tokenizer.js";++// TS mirror of tests/test_truncation.py (Python #181, issue #174): the state clamp must be+// reported from the token budget that actually applies, not guessed from character counts.++/** One token per whitespace-separated word, so token counts are exact and readable. */+const TOK: TokenizerLike = {+ clsId: 2,+ sepId: 3,+ maskId: 1,+ padId: 0,+ maskToken: "[MASK]",+ encode: (text: string) => text.split(/\s+/).filter(Boolean).map((w) => 10 + (w.length % 50)),+};++const Q = { t: "noul", ins: "Does the user ask for a refund?", crit: null } as const;+const LONG = Array.from({ length: 4000 }, (_, i) => `word${i}`).join(" ");++const statsFor = (state: string, maxLen = 512, headMaxLen = 192,+ truncateLeft = false, q: typeof Q | Record<string, unknown> = Q) =>+ buildSequence(TOK, state, q as never, maxLen, headMaxLen, undefined, truncateLeft).stats;++describe("sequence stats: the reported gap — a silent clamp (issue #174)", () => {+ it("a state that fits loses nothing, and says so", () => {+ const short = statsFor("the customer was billed twice");+ expect(short.truncated).toBe(false);+ expect(short.state_tokens_dropped).toBe(0);+ expect(short.state_tokens_used).toBe(short.state_tokens);+ });++ it("a state that does not fit reports the exact shortfall", () => {+ const clamped = statsFor(LONG);+ expect(clamped.truncated).toBe(true);+ expect(clamped.state_tokens).toBe(4000);+ expect(clamped.state_tokens_used).toBeLessThan(4000);+ expect(clamped.state_tokens_dropped).toBe(4000 - clamped.state_tokens_used);+ expect(clamped.state_tokens_used).toBeLessThanOrEqual(512);+ });+});++describe("the flag tracks the real window", () => {+ it("a 1024-token window keeps strictly more than a 512-token one", () => {+ const w512 = statsFor(LONG, 512);+ const w1024 = statsFor(LONG, 1024);+ expect(w1024.state_tokens_used).toBeGreaterThan(w512.state_tokens_used);+ expect(w1024.state_tokens_dropped).toBeLessThan(w512.state_tokens_dropped);+ });++ it("a mid-sized state is truncated on one checkpoint and intact on the other", () => {+ const mid = Array.from({ length: 600 }, (_, i) => `word${i}`).join(" ");+ expect(statsFor(mid, 512).truncated).toBe(true);+ expect(statsFor(mid, 1024).truncated).toBe(false);+ });++ it("a wider head leaves less room for the state", () => {+ const wide = {+ t: "choice",+ ins: "Which team should own this request, given the account tier and the contract terms?",+ crit: {+ billing: "invoices, payments, refunds, chargebacks and duplicate charges",+ technical: "bugs, outages, degraded performance and system errors",+ sales: "pricing, quotes, renewals and new contracts",+ other: "everything that does not belong to the three above",+ },+ };+ const narrow = { t: "noul", ins: "Refund?", crit: null };+ expect(statsFor(LONG, 512, 192, false, wide).state_tokens_used)+ .toBeLessThan(statsFor(LONG, 512, 192, false, narrow).state_tokens_used);+ });+});++describe("left truncation", () => {+ it("keeps as much as the right-hand clamp", () => {+ const left = statsFor(LONG, 512, 192, true);+ expect(left.truncated).toBe(true);+ expect(left.state_tokens_used).toBe(statsFor(LONG, 512).state_tokens_used);+ });++ it("with no room at all, slice(-0) is the whole state — count the final clamp instead", () => {+ const noRoom = statsFor(LONG, 8, 8, true);+ expect(noRoom.truncated).toBe(true);+ expect(noRoom.state_tokens_used).toBeLessThanOrEqual(8);+ expect(noRoom.state_tokens_dropped).toBe(noRoom.state_tokens - noRoom.state_tokens_used);+ });+});++describe("reported counts match the sequence", () => {+ it.each([+ ["512", 512, 192],+ ["1024", 1024, 512],+ ["tight", 64, 32],+ ] as const)("accounting/%s", (_name, maxLen, headMaxLen) => {+ const { ids, stats } = buildSequence(TOK, LONG, Q as never, maxLen, headMaxLen);+ expect(ids.length).toBeLessThanOrEqual(maxLen);++describe("systemOne reports truncation in usage", () => {+ const provider = () => ({+ async runEncoder(b: { inputIds: number[][] }) {+ return { lastHidden: b.inputIds.map((row) => row.map(() => 0)) };+ },+ async runHead(_h: unknown, b: { inputIds: number[][] }) {+ return {+ logits: b.inputIds.map(() => [0, 2]),+ act: b.inputIds.map(() => [1, 0]),+ };+ },+ });+ const agent = () =>+ new Agent({ provider: provider(), tok: TOK, max_len: 64, head_max_len: 48 } as never);+ const refund = { refund: { type: "noul", instructions: "Refund?" } } as never;++ it("publishes the flag, the dropped count and the affected questions", async () => {+ const long = Array.from({ length: 200 }, (_, i) => `w${i}`).join(" ");+ const r = await agent().systemOne(long, refund);+ expect(r.usage.input_tokens).toBeGreaterThan(0);+ expect(r.usage.output_tokens).toBe(0);+ expect(r.usage.state_tokens).toBe(200);+ expect(r.usage.truncated).toBe(true);+ expect(r.usage.state_tokens_dropped).toBeGreaterThan(0);+ expect((r.usage.state_tokens ?? 0) - (r.usage.state_tokens_dropped ?? 0))+ .toBeLessThanOrEqual(64);+ expect(r.usage.truncated_questions).toEqual(["refund"]);+ });++ it("a state that fits reports truncated: false", async () => {+ const r = await agent().systemOne("the customer was billed twice", refund);+ expect(r.usage.truncated).toBe(false);+ expect(r.usage.state_tokens_dropped).toBe(0);+ expect(r.usage.truncated_questions).toEqual([]);+ });++ it("questions over one state can disagree — only the starved head is named", async () => {+ const wideIns = "Which team should own this request given the account tier and contract terms?";+ const fortyWords = Array.from({ length: 40 }, (_, i) => `w${i}`).join(" ");+ const r = await agent().systemOne(fortyWords, {+ wide: { type: "choice", instructions: wideIns,+ criteria: { billing: "invoices and payments", technical: "bugs and outages",+ sales: "pricing and quotes", other: "everything else here" } },+ narrow: { type: "noul", instructions: "Refund?" },+ } as never);+ expect(r.usage.truncated).toBe(true);+ expect(r.usage.truncated_questions).toEqual(["wide"]);+ });++ it("predictBatch reports per state, aligned by index", async () => {+ const long = Array.from({ length: 200 }, (_, i) => `w${i}`).join(" ");+ const out = await agent().predictBatch(["short state here", long], refund);+ expect(out[0].usage.truncated).toBe(false);+ expect(out[1].usage.truncated).toBe(true);+ expect(out[1].usage.state_tokens).toBe(200);+ });++ it("empty questions keep the bare usage shape (no state ever encoded)", async () => {+ const r = await agent().systemOne("hi", {});+ expect(r.usage).toEqual({ input_tokens: 0, output_tokens: 0 });+ });+});++ expect(stats.state_tokens_used).toBeLessThanOrEqual(ids.length);+ expect(stats.state_tokens_used + stats.state_tokens_dropped).toBe(stats.state_tokens);+ expect(stats.truncated).toBe(stats.state_tokens_dropped > 0);+ });+});