Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): report state truncation in usage from the real token budget

Aashisha4f4c41f1b15
diff --git a/laya-ts/README.md b/laya-ts/README.md
index 5e4a6148..8d6e7c23 100644
--- a/laya-ts/README.md
+++ b/laya-ts/README.md
@@ -98,6 +98,23 @@ instead of a schema to get raw answers. Zod/TypeBox users can pass `z.toJSONSche
any object with a `toJSONSchema()` method is accepted. `planFromJsonSchema`,
`questionsFromJsonSchema` and `answersToJson` expose the planning and projection steps.
+## Truncation reporting
+
+The state is clamped to whatever token room a question's head leaves, and that budget moves
+with `max_len`, `head_max_len` and the rendered head — so `usage` reports it instead of letting
+you guess from character counts (issue #174; mirrors Python #181):
+
+```ts
+const out = await agent.predict(longState, questions);
+out.usage.truncated; // true when any state token was dropped
+out.usage.state_tokens; // encoded length of the full state
+out.usage.state_tokens_dropped; // worst case across the questions
+out.usage.truncated_questions; // ids of the questions whose head left too little room
+```
+
+The fields are absent only where no state was ever encoded (empty question schema, or a
+start hook that supplied the result).
+
## Batching (many states, one call)
Port of `Agent.predict_batch` / `Router.route_batch` / `Router.predict_batch`. The throughput
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index d4635316..7f2e4e3b 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -13,6 +13,7 @@ import {
softmax,
tempBucket,
} from "./common.js";
+import type { SequenceStats } from "./common.js";
import type { Batch, SessionProvider } from "./providers.js";
import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";
import { decide, type DecideOptions, type DecisionResult } from "./structured.js";
@@ -73,6 +74,13 @@ export type SystemAnswer = ChoiceAnswer | ScoreAnswer | NoulAnswer;
export interface SystemUsage {
input_tokens: number;
output_tokens: number;
+ /** State-fit reporting (issue #174, mirrors Python #181). Present on inference results;
+ * absent on hook-supplied (skipped) results and empty-question early returns. */
+ state_tokens?: number;
+ /** Worst case across the questions, which share one state but not one head budget. */
+ state_tokens_dropped?: number;
+ truncated?: boolean;
+ truncated_questions?: string[];
}
export interface SystemOneResult {
@@ -428,28 +436,30 @@ export class Agent extends HookRegistry {
internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[],
maxLen: number,
headMaxLen: number,
- ): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number } {
+ ): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number; stats: SequenceStats[] } {
// The state text is shared by every question and is usually the longest text in the
// sequence — encode it once and compose the per-question prefixes onto it.
const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));
const items: { ids: number[]; markers: number[]; qtype: number }[] = [];
+ const stats: SequenceStats[] = [];
for (let qi = 0; qi < ids.length; qi++) {
const qid = ids[qi];
const q = internals[qi];
const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen);
- const { ids: seq, markers } = sequenceWithState(
+ const seq = sequenceWithState(
prefix,
stAll,
this.tok.sepId,
maxLen,
Array.isArray(state),
);
- if (markers.length !== renderOptions(q).length) {
+ if (seq.markers.length !== renderOptions(q).length) {
throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${headMaxLen}`);
}
- items.push({ ids: seq, markers, qtype: QTYPES[q.t] });
+ items.push({ ids: seq.ids, markers: seq.markers, qtype: QTYPES[q.t] });
+ stats.push(seq.stats);
}
- return { items, nTokens: items.reduce((a, it) => a + it.ids.length, 0) };
+ return { items, nTokens: items.reduce((a, it) => a + it.ids.length, 0), stats };
}
/**
@@ -526,10 +536,22 @@ export class Agent extends HookRegistry {
);
row++;
}
+ // How much of the state actually reached the encoder. Truncation is a token budget, so it
+ // moves with maxLen, headMaxLen and each question's rendered head — a caller cannot infer
+ // it from the length of the state it sent (issue #174, Python #181 parity).
+ const st = built[s].stats;
+ const dropped = st.reduce((a, x) => Math.max(a, x.state_tokens_dropped), 0);
out.push({
model: "laya-rl-agent",
answers,
- usage: { input_tokens: built[s].nTokens, output_tokens: 0 },
+ usage: {
+ input_tokens: built[s].nTokens,
+ output_tokens: 0,
+ state_tokens: st[0]?.state_tokens ?? 0,
+ state_tokens_dropped: dropped,
+ truncated: dropped > 0,
+ truncated_questions: ids.filter((_, qi) => st[qi].truncated),
+ },
});
}
}
diff --git a/laya-ts/src/common.ts b/laya-ts/src/common.ts
index ae5b9da8..94fb9f01 100644
--- a/laya-ts/src/common.ts
+++ b/laya-ts/src/common.ts
@@ -96,19 +96,45 @@ export function buildQuestionPrefix(tok: TokenizerLike, q: InternalQ,
ids.push(tok.sepId);
return { ids, markers, nOptions: opts.length };
}
+export interface SequenceStats {
+ /** Encoded length of the full state, before the window clamp. */
+ state_tokens: number;
+ /** State tokens that actually reached the encoder after the final maxLen clamp. */
+ state_tokens_used: number;
+ /** State tokens dropped by the clamp: state_tokens - state_tokens_used. */
+ state_tokens_dropped: number;
+ /** True when the clamp dropped any state token. */
+ truncated: boolean;
+}
+
/** Append pre-encoded state tokens to a question prefix. Identical output to building the
* whole sequence in one pass, but the state only needs encoding once per state, not once
- * per (state, question) pair. */
+ * per (state, question) pair.
+ *
+ * The state is clamped to whatever room the head leaves; `stats` reports that clamp so callers
+ * never have to guess it from the character length of what they sent (issue #174, Python #181). */
export function sequenceWithState(prefix: QuestionPrefix, stateIds: number[], sepId: number,
- maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[] } {
+ maxLen = 512, truncateLeft = false): { ids: number[]; markers: number[]; stats: SequenceStats } {
const room = Math.max(0, maxLen - prefix.ids.length - 1);
// not stateIds.slice(-room): with no room left, slice(-0) is the whole state rather than none of it
- const st = truncateLeft ? stateIds.slice(Math.max(0, stateIds.length - room)) : stateIds.slice(0, room);
- const ids = [...prefix.ids, ...st, sepId].slice(0, maxLen);
- return { ids, markers: prefix.markers.filter((m) => m < maxLen) };
+ const kept = truncateLeft ? stateIds.slice(Math.max(0, stateIds.length - room)) : stateIds.slice(0, room);
+ const ids = [...prefix.ids, ...kept, sepId].slice(0, maxLen);
+ // Count against the final clamp rather than `kept`: the clamp is what actually decided
+ // which state tokens reached the encoder.
+ const used = Math.max(0, Math.min(kept.length, maxLen - prefix.ids.length));
+ return {
+ ids,
+ markers: prefix.markers.filter((m) => m < maxLen),
+ stats: {
+ state_tokens: stateIds.length,
+ state_tokens_used: used,
+ state_tokens_dropped: stateIds.length - used,
+ truncated: used < stateIds.length,
+ },
+ };
}
export function buildSequence(tok: TokenizerLike, state: unknown, q: InternalQ,
- maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[] } {
+ maxLen = 512, headMaxLen = 192, optionOrder?: number[], truncateLeft = false): { ids: number[]; markers: number[]; stats: SequenceStats } {
const stAll = tok.encode(serializeState(state).split(tok.maskToken).join(" "));
return sequenceWithState(buildQuestionPrefix(tok, q, maxLen, headMaxLen, optionOrder), stAll, tok.sepId, maxLen, truncateLeft);
}
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index b311d389..25896c67 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -64,6 +64,6 @@ export {
TEMP_MIN,
TEMP_MAX,
} from "./common.js";
-export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix } from "./common.js";
+export type { QType, InternalQ, CollateItem, CollatedBatch, QuestionPrefix, SequenceStats } from "./common.js";
export { bpeEncode, metaspaceEncode, encodeWithData, parseTokenizerJson, loadTokenizerJson, CHECKPOINT_IDS, SPECIAL_ALIASES, METASPACE_REPLACEMENT } from "./tokenizer.js";
export type { TokenizerLike, TokenizerData, TokenizerIds, PreTokenizerKind } from "./tokenizer.js";
diff --git a/laya-ts/tests/truncation.test.ts b/laya-ts/tests/truncation.test.ts
new file mode 100644
index 00000000..4e3aa077
--- /dev/null
+++ b/laya-ts/tests/truncation.test.ts
@@ -0,0 +1,166 @@
+import { describe, expect, it } from "vitest";
+import { Agent } from "../src/agent.js";
+import { buildSequence } from "../src/common.js";
+import type { TokenizerLike } from "../src/tokenizer.js";
+
+// TS mirror of tests/test_truncation.py (Python #181, issue #174): the state clamp must be
+// reported from the token budget that actually applies, not guessed from character counts.
+
+/** One token per whitespace-separated word, so token counts are exact and readable. */
+const TOK: TokenizerLike = {
+ clsId: 2,
+ sepId: 3,
+ maskId: 1,
+ padId: 0,
+ maskToken: "[MASK]",
+ encode: (text: string) => text.split(/\s+/).filter(Boolean).map((w) => 10 + (w.length % 50)),
+};
+
+const Q = { t: "noul", ins: "Does the user ask for a refund?", crit: null } as const;
+const LONG = Array.from({ length: 4000 }, (_, i) => `word${i}`).join(" ");
+
+const statsFor = (state: string, maxLen = 512, headMaxLen = 192,
+ truncateLeft = false, q: typeof Q | Record<string, unknown> = Q) =>
+ buildSequence(TOK, state, q as never, maxLen, headMaxLen, undefined, truncateLeft).stats;
+
+describe("sequence stats: the reported gap — a silent clamp (issue #174)", () => {
+ it("a state that fits loses nothing, and says so", () => {
+ const short = statsFor("the customer was billed twice");
+ expect(short.truncated).toBe(false);
+ expect(short.state_tokens_dropped).toBe(0);
+ expect(short.state_tokens_used).toBe(short.state_tokens);
+ });
+
+ it("a state that does not fit reports the exact shortfall", () => {
+ const clamped = statsFor(LONG);
+ expect(clamped.truncated).toBe(true);
+ expect(clamped.state_tokens).toBe(4000);
+ expect(clamped.state_tokens_used).toBeLessThan(4000);
+ expect(clamped.state_tokens_dropped).toBe(4000 - clamped.state_tokens_used);
+ expect(clamped.state_tokens_used).toBeLessThanOrEqual(512);
+ });
+});
+
+describe("the flag tracks the real window", () => {
+ it("a 1024-token window keeps strictly more than a 512-token one", () => {
+ const w512 = statsFor(LONG, 512);
+ const w1024 = statsFor(LONG, 1024);
+ expect(w1024.state_tokens_used).toBeGreaterThan(w512.state_tokens_used);
+ expect(w1024.state_tokens_dropped).toBeLessThan(w512.state_tokens_dropped);
+ });
+
+ it("a mid-sized state is truncated on one checkpoint and intact on the other", () => {
+ const mid = Array.from({ length: 600 }, (_, i) => `word${i}`).join(" ");
+ expect(statsFor(mid, 512).truncated).toBe(true);
+ expect(statsFor(mid, 1024).truncated).toBe(false);
+ });
+
+ it("a wider head leaves less room for the state", () => {
+ const wide = {
+ t: "choice",
+ ins: "Which team should own this request, given the account tier and the contract terms?",
+ crit: {
+ billing: "invoices, payments, refunds, chargebacks and duplicate charges",
+ technical: "bugs, outages, degraded performance and system errors",
+ sales: "pricing, quotes, renewals and new contracts",
+ other: "everything that does not belong to the three above",
+ },
+ };
+ const narrow = { t: "noul", ins: "Refund?", crit: null };
+ expect(statsFor(LONG, 512, 192, false, wide).state_tokens_used)
+ .toBeLessThan(statsFor(LONG, 512, 192, false, narrow).state_tokens_used);
+ });
+});
+
+describe("left truncation", () => {
+ it("keeps as much as the right-hand clamp", () => {
+ const left = statsFor(LONG, 512, 192, true);
+ expect(left.truncated).toBe(true);
+ expect(left.state_tokens_used).toBe(statsFor(LONG, 512).state_tokens_used);
+ });
+
+ it("with no room at all, slice(-0) is the whole state — count the final clamp instead", () => {
+ const noRoom = statsFor(LONG, 8, 8, true);
+ expect(noRoom.truncated).toBe(true);
+ expect(noRoom.state_tokens_used).toBeLessThanOrEqual(8);
+ expect(noRoom.state_tokens_dropped).toBe(noRoom.state_tokens - noRoom.state_tokens_used);
+ });
+});
+
+describe("reported counts match the sequence", () => {
+ it.each([
+ ["512", 512, 192],
+ ["1024", 1024, 512],
+ ["tight", 64, 32],
+ ] as const)("accounting/%s", (_name, maxLen, headMaxLen) => {
+ const { ids, stats } = buildSequence(TOK, LONG, Q as never, maxLen, headMaxLen);
+ expect(ids.length).toBeLessThanOrEqual(maxLen);
+
+describe("systemOne reports truncation in usage", () => {
+ const provider = () => ({
+ async runEncoder(b: { inputIds: number[][] }) {
+ return { lastHidden: b.inputIds.map((row) => row.map(() => 0)) };
+ },
+ async runHead(_h: unknown, b: { inputIds: number[][] }) {
+ return {
+ logits: b.inputIds.map(() => [0, 2]),
+ act: b.inputIds.map(() => [1, 0]),
+ };
+ },
+ });
+ const agent = () =>
+ new Agent({ provider: provider(), tok: TOK, max_len: 64, head_max_len: 48 } as never);
+ const refund = { refund: { type: "noul", instructions: "Refund?" } } as never;
+
+ it("publishes the flag, the dropped count and the affected questions", async () => {
+ const long = Array.from({ length: 200 }, (_, i) => `w${i}`).join(" ");
+ const r = await agent().systemOne(long, refund);
+ expect(r.usage.input_tokens).toBeGreaterThan(0);
+ expect(r.usage.output_tokens).toBe(0);
+ expect(r.usage.state_tokens).toBe(200);
+ expect(r.usage.truncated).toBe(true);
+ expect(r.usage.state_tokens_dropped).toBeGreaterThan(0);
+ expect((r.usage.state_tokens ?? 0) - (r.usage.state_tokens_dropped ?? 0))
+ .toBeLessThanOrEqual(64);
+ expect(r.usage.truncated_questions).toEqual(["refund"]);
+ });
+
+ it("a state that fits reports truncated: false", async () => {
+ const r = await agent().systemOne("the customer was billed twice", refund);
+ expect(r.usage.truncated).toBe(false);
+ expect(r.usage.state_tokens_dropped).toBe(0);
+ expect(r.usage.truncated_questions).toEqual([]);
+ });
+
+ it("questions over one state can disagree — only the starved head is named", async () => {
+ const wideIns = "Which team should own this request given the account tier and contract terms?";
+ const fortyWords = Array.from({ length: 40 }, (_, i) => `w${i}`).join(" ");
+ const r = await agent().systemOne(fortyWords, {
+ wide: { type: "choice", instructions: wideIns,
+ criteria: { billing: "invoices and payments", technical: "bugs and outages",
+ sales: "pricing and quotes", other: "everything else here" } },
+ narrow: { type: "noul", instructions: "Refund?" },
+ } as never);
+ expect(r.usage.truncated).toBe(true);
+ expect(r.usage.truncated_questions).toEqual(["wide"]);
+ });
+
+ it("predictBatch reports per state, aligned by index", async () => {
+ const long = Array.from({ length: 200 }, (_, i) => `w${i}`).join(" ");
+ const out = await agent().predictBatch(["short state here", long], refund);
+ expect(out[0].usage.truncated).toBe(false);
+ expect(out[1].usage.truncated).toBe(true);
+ expect(out[1].usage.state_tokens).toBe(200);
+ });
+
+ it("empty questions keep the bare usage shape (no state ever encoded)", async () => {
+ const r = await agent().systemOne("hi", {});
+ expect(r.usage).toEqual({ input_tokens: 0, output_tokens: 0 });
+ });
+});
+
+ expect(stats.state_tokens_used).toBeLessThanOrEqual(ids.length);
+ expect(stats.state_tokens_used + stats.state_tokens_dropped).toBe(stats.state_tokens);
+ expect(stats.truncated).toBe(stats.state_tokens_dropped > 0);
+ });
+});