Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): forward batch budgets through the single-call fallback

xinpingzh75033a42a0bc
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index 4b7d9d23..0e7ac03b 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -934,7 +934,7 @@ export class Router extends HookRegistry {
systemOne(
state: unknown,
questions: Record<string, QuestionDef>,
- opts?: { lang?: string | null },
+ opts?: { lang?: string | null; maxLen?: number | null; headMaxLen?: number | null },
): Promise<SystemOneResult>;
langTemperatures?: Record<string, unknown>;
};
@@ -1037,7 +1037,10 @@ export class Router extends HookRegistry {
// still work; their states simply share no forward pass.
batchResults = [];
for (const [, ctx] of group.items) {
- const singleOpts = group.lang === null ? {} : { lang: group.lang };
+ const singleOpts: { lang?: string; maxLen?: number; headMaxLen?: number } =
+ group.lang === null ? {} : { lang: group.lang };
+ if (group.maxLen !== null) singleOpts.maxLen = group.maxLen;
+ if (group.headMaxLen !== null) singleOpts.headMaxLen = group.headMaxLen;
markDefaultsRan(singleOpts);
batchResults.push(
await agent.systemOne(
diff --git a/laya-ts/tests/router-batch.test.ts b/laya-ts/tests/router-batch.test.ts
index 32503e73..33ab9d7b 100644
--- a/laya-ts/tests/router-batch.test.ts
+++ b/laya-ts/tests/router-batch.test.ts
@@ -568,6 +568,37 @@ describe("Router.routeBatch / predictBatch", () => {
).rejects.toThrow(/returned 0 results for 2 states/);
});
+ it.each([false, true])("forwards fallback budgets with hook override=%s", async (override) => {
+ const calls: unknown[] = [];
+ const router = new Router({
+ loader: (() => ({
+ langTemperatures: { en: {} },
+ async systemOne(state: unknown, _questions: unknown, opts: unknown) {
+ calls.push([state, opts]);
+ return { model: "stub", answers: { seen: state }, usage: {} };
+ },
+ })) as never,
+ });
+ const results = await router.predictBatch([
+ req("budgeted", { model: "english", lang: "en", maxLen: 64, headMaxLen: 16 }),
+ req("defaults", { model: "english", lang: "en", maxLen: null, headMaxLen: null }),
+ req("zero", { model: "english", lang: "en", maxLen: 0, headMaxLen: 0 }),
+ ], 2, {
+ onPredictStart(ctx) {
+ if (override && ctx.states[0] === "budgeted") {
+ ctx.maxLen = 96;
+ ctx.headMaxLen = 24;
+ }
+ },
+ });
+ expect(calls).toEqual([
+ ["budgeted", { lang: "en", maxLen: override ? 96 : 64, headMaxLen: override ? 24 : 16 }],
+ ["defaults", { lang: "en" }],
+ ["zero", { lang: "en", maxLen: 0, headMaxLen: 0 }],
+ ]);
+ expect(results.map((r) => r.answers.seen)).toEqual(["budgeted", "defaults", "zero"]);
+ });
+
it("falls back to systemOne for agent-like objects without predictBatch", async () => {
const router = new Router({
loader: (() => ({
diff --git a/laya-ts/tests/structured.test.ts b/laya-ts/tests/structured.test.ts
index 8765688e..262160c5 100644
--- a/laya-ts/tests/structured.test.ts
+++ b/laya-ts/tests/structured.test.ts
@@ -644,6 +644,40 @@ describe("structured/decideBatch", () => {
+describe("structured/decideBatch Router token budgets", () => {
+ it.each([32, 512])("matches single encoding when overriding the Agent default of %s", async (maxLen) => {
+ const encoderLengths: number[][] = [];
+ const agent = new Agent({
+ max_len: maxLen,
+ provider: {
+ async runEncoder(batch) {
+ encoderLengths.push(batch.inputIds.map((row) => row.length));
+ return { lastHidden: batch.inputIds.map(() => [[1, 0], [0, 1]]) };
+ },
+ async runHead(_hidden, batch) {
+ return {
+ logits: batch.inputIds.map(() => [2, 0]),
+ act: batch.inputIds.map(() => [3, 0]),
+ };
+ },
+ },
+ });
+ const router = new Router({ default: "english", loader: () => agent });
+ const schema = {
+ type: "object",
+ properties: { relevant: { type: "boolean", description: "Is the request relevant?" } },
+ };
+ const state = Array.from({ length: 160 }, (_, i) => `word${i}`).join(" ");
+ const opts = { model: "english", maxLen: 64, headMaxLen: 16, returnDetails: true as const };
+ const single = await router.decide(state, schema, opts);
+ const batch = await router.decideBatch([state, state], schema, opts);
+ expect(single.usage).toMatchObject({ input_tokens: 64, state_tokens_dropped: 114, truncated: true });
+ expect(batch.map((result) => result.usage)).toEqual([single.usage, single.usage]);
+ expect(batch.map((result) => result.values)).toEqual([single.values, single.values]);
+ expect(encoderLengths).toEqual([[64], [64, 64]]);
+ });
+});
+
// `DecideBatchOptions` documents that anything it does not read itself is forwarded to the
// runner, hooks included. On a Router-like runner the call used to spread those options into the
// request objects and pass no third argument, so `Router.predictBatch` -- which reads hooks from