Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): honor per-request budgets in routed batches
xinpingzhc70bc16635da
diff --git a/laya-ts/README.md b/laya-ts/README.mdindex 0448a4e1..fee7cf6f 100644--- a/laya-ts/README.md+++ b/laya-ts/README.md@@ -305,6 +305,10 @@ const results = await agent.predictBatch(states, questions, { batchSize: 32 });// in as few forward passes as possible. Results keep input order and carry `routing`.const decisions = router.routeBatch(requests); // validate + route, nothing loadsconst routed = await router.predictBatch(requests, 32); // == router.predictMany(...)+// Token budgets belong to each request; omitted/null values use the Agent config.+const budgeted = await router.predictBatch([+ { state: "ticket", questions, maxLen: 64, headMaxLen: 16 },+]);// Per-call hooks are composed like predict: installed hooks first, then this list,// then onPredictStart / onPredictEnd. null and [] add nothing. hooksRaise overrides// the Router's policy for this call. The list runs once per request, including onRoute.@@ -315,8 +319,8 @@ const traced = await router.predictBatch(requests, null, {});// requests routed to the same checkpoint still split into separate batches when their-// question schemas differ (order-sensitively), when per-request start hooks set-// different ctx.maxLen / ctx.headMaxLen overrides, or — for an agent carrying+// question schemas differ (order-sensitively), when their maxLen / headMaxLen+// budgets differ after start hooks (which can replace or clear them), or — for an agent carrying// lang_temperatures — when their languages differ: each request's effective language (an// explicit lang, otherwise the detected non-English one) is forwarded to the agent, so// the batched path scores exactly like predict. Router-level predict hooks run once perdiff --git a/laya-ts/src/router.ts b/laya-ts/src/router.tsindex 5a85b104..4b7d9d23 100644--- a/laya-ts/src/router.ts+++ b/laya-ts/src/router.ts@@ -189,7 +189,7 @@ export interface PredictBatchOptions {hooksRaise?: boolean;}-/** One item of a Router.predictBatch/routeBatch batch: state, questions, route overrides. */+/** One item of a Router.predictBatch/routeBatch batch: state, questions, route and budget overrides. */export interface BatchRequest {state: unknown;questions: Record<string, QuestionDef>;@@ -198,6 +198,9 @@ export interface BatchRequest {lang?: string | null;langGuess?: LangGuess;lang_guess?: LangGuess;+ /** Per-request token budgets; null = agent config. Start hooks may override them. */+ maxLen?: number | null;+ headMaxLen?: number | null;}function questionSchema(questions: Record<string, unknown>): string {@@ -945,6 +948,8 @@ export class Router extends HookRegistry {model: modelName,agent,router: this,+ maxLen: requests[i].maxLen ?? null,+ headMaxLen: requests[i].headMaxLen ?? null,});started.push(ctx);dispatch(active, "onPredictStart", ctx, { raiseErrors });diff --git a/laya-ts/tests/router-batch.test.ts b/laya-ts/tests/router-batch.test.tsindex 163fb392..32503e73 100644--- a/laya-ts/tests/router-batch.test.ts+++ b/laya-ts/tests/router-batch.test.ts@@ -498,6 +498,64 @@ describe("Router.routeBatch / predictBatch", () => {expect(calls[1][3]).toEqual({ batchSize: null, maxLen: 64 });});+ it("seeds request budgets before hooks and groups by both final budgets", async () => {+ const { router, calls } = makeRouter();+ const seen: unknown[] = [];+ const results = await router.predictBatch([+ req("a", { model: "english", maxLen: 64, headMaxLen: 16 }),+ req("b", { model: "english", maxLen: 128, headMaxLen: 16 }),+ req("c", { model: "english", maxLen: 64, headMaxLen: 32 }),+ req("d", { model: "english", maxLen: 64, headMaxLen: 16 }),+ ], 2, {+ onPredictStart(ctx) { seen.push([ctx.maxLen, ctx.headMaxLen]); },+ });+ expect(seen).toEqual([[64, 16], [128, 16], [64, 32], [64, 16]]);+ expect(calls.map((c) => [c[1], c[3]])).toEqual([+ [["a", "d"], { batchSize: 2, maxLen: 64, headMaxLen: 16 }],+ [["b"], { batchSize: 2, maxLen: 128, headMaxLen: 16 }],+ [["c"], { batchSize: 2, maxLen: 64, headMaxLen: 32 }],+ ]);+ expect(results.map((r) => r.answers.seen)).toEqual(["a", "b", "c", "d"]);+ });++ it("lets hooks replace or clear explicit budgets before grouping", async () => {+ const { router, calls } = makeRouter();+ const seen: unknown[] = [];+ await router.predictBatch([+ req("replace", { model: "english", maxLen: 64, headMaxLen: 16 }),+ req("clear", { model: "english", maxLen: 128, headMaxLen: 32 }),+ req("defaults", { model: "english" }),+ ], null, {+ onPredictStart(ctx) {+ seen.push([ctx.maxLen, ctx.headMaxLen]);+ if (ctx.states[0] === "replace") {+ ctx.maxLen = 96;+ ctx.headMaxLen = 24;+ } else if (ctx.states[0] === "clear") {+ ctx.maxLen = ctx.headMaxLen = null;+ }+ },+ });+ expect(seen).toEqual([[64, 16], [128, 32], [null, null]]);+ expect(calls.map((c) => [c[1], c[3]])).toEqual([+ [["replace"], { batchSize: null, maxLen: 96, headMaxLen: 24 }],+ [["clear", "defaults"], { batchSize: null }],+ ]);+ });++ it("keeps absent and null budgets unset without dropping zero overrides", async () => {+ const { router, calls } = makeRouter();+ await router.predictBatch([+ req("absent", { model: "english" }),+ req("null", { model: "english", maxLen: null, headMaxLen: null }),+ req("zero", { model: "english", maxLen: 0, headMaxLen: 0 }),+ ]);+ expect(calls.map((c) => [c[1], c[3]])).toEqual([+ [["absent", "null"], { batchSize: null }],+ [["zero"], { batchSize: null, maxLen: 0, headMaxLen: 0 }],+ ]);+ });+it("raises when the agent returns the wrong number of results", async () => {const router = new Router({loader: (() => ({diff --git a/laya-ts/tests/types/router-batch-budgets.types.ts b/laya-ts/tests/types/router-batch-budgets.types.tsnew file mode 100644index 00000000..513f64ef--- /dev/null+++ b/laya-ts/tests/types/router-batch-budgets.types.ts@@ -0,0 +1,10 @@+import type { BatchRequest } from "../../src/index.js";++export const requests: BatchRequest[] = [+ { state: "ticket", questions: {}, maxLen: 64, headMaxLen: 16 },+ { state: "defaults", questions: {}, maxLen: null, headMaxLen: null },+ { state: "omitted", questions: {} },+];++// @ts-expect-error token budgets must be numbers or null+export const invalid: BatchRequest = { state: "ticket", questions: {}, maxLen: "64" };