Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): honor per-request budgets in routed batches

xinpingzhc70bc16635da
diff --git a/laya-ts/README.md b/laya-ts/README.md
index 0448a4e1..fee7cf6f 100644
--- a/laya-ts/README.md
+++ b/laya-ts/README.md
@@ -305,6 +305,10 @@ const results = await agent.predictBatch(states, questions, { batchSize: 32 });
// in as few forward passes as possible. Results keep input order and carry `routing`.
const decisions = router.routeBatch(requests); // validate + route, nothing loads
const routed = await router.predictBatch(requests, 32); // == router.predictMany(...)
+// Token budgets belong to each request; omitted/null values use the Agent config.
+const budgeted = await router.predictBatch([
+ { state: "ticket", questions, maxLen: 64, headMaxLen: 16 },
+]);
// Per-call hooks are composed like predict: installed hooks first, then this list,
// then onPredictStart / onPredictEnd. null and [] add nothing. hooksRaise overrides
// the Router's policy for this call. The list runs once per request, including onRoute.
@@ -315,8 +319,8 @@ const traced = await router.predictBatch(requests, null, {
});
// requests routed to the same checkpoint still split into separate batches when their
-// question schemas differ (order-sensitively), when per-request start hooks set
-// different ctx.maxLen / ctx.headMaxLen overrides, or — for an agent carrying
+// question schemas differ (order-sensitively), when their maxLen / headMaxLen
+// budgets differ after start hooks (which can replace or clear them), or — for an agent carrying
// lang_temperatures — when their languages differ: each request's effective language (an
// explicit lang, otherwise the detected non-English one) is forwarded to the agent, so
// the batched path scores exactly like predict. Router-level predict hooks run once per
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index 5a85b104..4b7d9d23 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -189,7 +189,7 @@ export interface PredictBatchOptions {
hooksRaise?: boolean;
}
-/** One item of a Router.predictBatch/routeBatch batch: state, questions, route overrides. */
+/** One item of a Router.predictBatch/routeBatch batch: state, questions, route and budget overrides. */
export interface BatchRequest {
state: unknown;
questions: Record<string, QuestionDef>;
@@ -198,6 +198,9 @@ export interface BatchRequest {
lang?: string | null;
langGuess?: LangGuess;
lang_guess?: LangGuess;
+ /** Per-request token budgets; null = agent config. Start hooks may override them. */
+ maxLen?: number | null;
+ headMaxLen?: number | null;
}
function questionSchema(questions: Record<string, unknown>): string {
@@ -945,6 +948,8 @@ export class Router extends HookRegistry {
model: modelName,
agent,
router: this,
+ maxLen: requests[i].maxLen ?? null,
+ headMaxLen: requests[i].headMaxLen ?? null,
});
started.push(ctx);
dispatch(active, "onPredictStart", ctx, { raiseErrors });
diff --git a/laya-ts/tests/router-batch.test.ts b/laya-ts/tests/router-batch.test.ts
index 163fb392..32503e73 100644
--- a/laya-ts/tests/router-batch.test.ts
+++ b/laya-ts/tests/router-batch.test.ts
@@ -498,6 +498,64 @@ describe("Router.routeBatch / predictBatch", () => {
expect(calls[1][3]).toEqual({ batchSize: null, maxLen: 64 });
});
+ it("seeds request budgets before hooks and groups by both final budgets", async () => {
+ const { router, calls } = makeRouter();
+ const seen: unknown[] = [];
+ const results = await router.predictBatch([
+ req("a", { model: "english", maxLen: 64, headMaxLen: 16 }),
+ req("b", { model: "english", maxLen: 128, headMaxLen: 16 }),
+ req("c", { model: "english", maxLen: 64, headMaxLen: 32 }),
+ req("d", { model: "english", maxLen: 64, headMaxLen: 16 }),
+ ], 2, {
+ onPredictStart(ctx) { seen.push([ctx.maxLen, ctx.headMaxLen]); },
+ });
+ expect(seen).toEqual([[64, 16], [128, 16], [64, 32], [64, 16]]);
+ expect(calls.map((c) => [c[1], c[3]])).toEqual([
+ [["a", "d"], { batchSize: 2, maxLen: 64, headMaxLen: 16 }],
+ [["b"], { batchSize: 2, maxLen: 128, headMaxLen: 16 }],
+ [["c"], { batchSize: 2, maxLen: 64, headMaxLen: 32 }],
+ ]);
+ expect(results.map((r) => r.answers.seen)).toEqual(["a", "b", "c", "d"]);
+ });
+
+ it("lets hooks replace or clear explicit budgets before grouping", async () => {
+ const { router, calls } = makeRouter();
+ const seen: unknown[] = [];
+ await router.predictBatch([
+ req("replace", { model: "english", maxLen: 64, headMaxLen: 16 }),
+ req("clear", { model: "english", maxLen: 128, headMaxLen: 32 }),
+ req("defaults", { model: "english" }),
+ ], null, {
+ onPredictStart(ctx) {
+ seen.push([ctx.maxLen, ctx.headMaxLen]);
+ if (ctx.states[0] === "replace") {
+ ctx.maxLen = 96;
+ ctx.headMaxLen = 24;
+ } else if (ctx.states[0] === "clear") {
+ ctx.maxLen = ctx.headMaxLen = null;
+ }
+ },
+ });
+ expect(seen).toEqual([[64, 16], [128, 32], [null, null]]);
+ expect(calls.map((c) => [c[1], c[3]])).toEqual([
+ [["replace"], { batchSize: null, maxLen: 96, headMaxLen: 24 }],
+ [["clear", "defaults"], { batchSize: null }],
+ ]);
+ });
+
+ it("keeps absent and null budgets unset without dropping zero overrides", async () => {
+ const { router, calls } = makeRouter();
+ await router.predictBatch([
+ req("absent", { model: "english" }),
+ req("null", { model: "english", maxLen: null, headMaxLen: null }),
+ req("zero", { model: "english", maxLen: 0, headMaxLen: 0 }),
+ ]);
+ expect(calls.map((c) => [c[1], c[3]])).toEqual([
+ [["absent", "null"], { batchSize: null }],
+ [["zero"], { batchSize: null, maxLen: 0, headMaxLen: 0 }],
+ ]);
+ });
+
it("raises when the agent returns the wrong number of results", async () => {
const router = new Router({
loader: (() => ({
diff --git a/laya-ts/tests/types/router-batch-budgets.types.ts b/laya-ts/tests/types/router-batch-budgets.types.ts
new file mode 100644
index 00000000..513f64ef
--- /dev/null
+++ b/laya-ts/tests/types/router-batch-budgets.types.ts
@@ -0,0 +1,10 @@
+import type { BatchRequest } from "../../src/index.js";
+
+export const requests: BatchRequest[] = [
+ { state: "ticket", questions: {}, maxLen: 64, headMaxLen: 16 },
+ { state: "defaults", questions: {}, maxLen: null, headMaxLen: null },
+ { state: "omitted", questions: {} },
+];
+
+// @ts-expect-error token budgets must be numbers or null
+export const invalid: BatchRequest = { state: "ticket", questions: {}, maxLen: "64" };