Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): forward batch budgets through the single-call fallback
xinpingzh75033a42a0bc
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.tsindex 4b7d9d23..0e7ac03b 100644--- a/laya-ts/src/router.ts+++ b/laya-ts/src/router.ts@@ -934,7 +934,7 @@ export class Router extends HookRegistry {systemOne(state: unknown,questions: Record<string, QuestionDef>,- opts?: { lang?: string | null },+ opts?: { lang?: string | null; maxLen?: number | null; headMaxLen?: number | null },): Promise<SystemOneResult>;langTemperatures?: Record<string, unknown>;};@@ -1037,7 +1037,10 @@ export class Router extends HookRegistry {// still work; their states simply share no forward pass.batchResults = [];for (const [, ctx] of group.items) {- const singleOpts = group.lang === null ? {} : { lang: group.lang };+ const singleOpts: { lang?: string; maxLen?: number; headMaxLen?: number } =+ group.lang === null ? {} : { lang: group.lang };+ if (group.maxLen !== null) singleOpts.maxLen = group.maxLen;+ if (group.headMaxLen !== null) singleOpts.headMaxLen = group.headMaxLen;markDefaultsRan(singleOpts);batchResults.push(await agent.systemOne(diff --git a/laya-ts/tests/router-batch.test.ts b/laya-ts/tests/router-batch.test.tsindex 32503e73..33ab9d7b 100644--- a/laya-ts/tests/router-batch.test.ts+++ b/laya-ts/tests/router-batch.test.ts@@ -568,6 +568,37 @@ describe("Router.routeBatch / predictBatch", () => {).rejects.toThrow(/returned 0 results for 2 states/);});+ it.each([false, true])("forwards fallback budgets with hook override=%s", async (override) => {+ const calls: unknown[] = [];+ const router = new Router({+ loader: (() => ({+ langTemperatures: { en: {} },+ async systemOne(state: unknown, _questions: unknown, opts: unknown) {+ calls.push([state, opts]);+ return { model: "stub", answers: { seen: state }, usage: {} };+ },+ })) as never,+ });+ const results = await router.predictBatch([+ req("budgeted", { model: "english", lang: "en", maxLen: 64, headMaxLen: 16 }),+ req("defaults", { model: "english", lang: "en", maxLen: null, headMaxLen: null }),+ req("zero", { model: "english", lang: "en", maxLen: 0, headMaxLen: 0 }),+ ], 2, {+ onPredictStart(ctx) {+ if (override && ctx.states[0] === "budgeted") {+ ctx.maxLen = 96;+ ctx.headMaxLen = 24;+ }+ },+ });+ expect(calls).toEqual([+ ["budgeted", { lang: "en", maxLen: override ? 96 : 64, headMaxLen: override ? 24 : 16 }],+ ["defaults", { lang: "en" }],+ ["zero", { lang: "en", maxLen: 0, headMaxLen: 0 }],+ ]);+ expect(results.map((r) => r.answers.seen)).toEqual(["budgeted", "defaults", "zero"]);+ });+it("falls back to systemOne for agent-like objects without predictBatch", async () => {const router = new Router({loader: (() => ({diff --git a/laya-ts/tests/structured.test.ts b/laya-ts/tests/structured.test.tsindex 8765688e..262160c5 100644--- a/laya-ts/tests/structured.test.ts+++ b/laya-ts/tests/structured.test.ts@@ -644,6 +644,40 @@ describe("structured/decideBatch", () => {+describe("structured/decideBatch Router token budgets", () => {+ it.each([32, 512])("matches single encoding when overriding the Agent default of %s", async (maxLen) => {+ const encoderLengths: number[][] = [];+ const agent = new Agent({+ max_len: maxLen,+ provider: {+ async runEncoder(batch) {+ encoderLengths.push(batch.inputIds.map((row) => row.length));+ return { lastHidden: batch.inputIds.map(() => [[1, 0], [0, 1]]) };+ },+ async runHead(_hidden, batch) {+ return {+ logits: batch.inputIds.map(() => [2, 0]),+ act: batch.inputIds.map(() => [3, 0]),+ };+ },+ },+ });+ const router = new Router({ default: "english", loader: () => agent });+ const schema = {+ type: "object",+ properties: { relevant: { type: "boolean", description: "Is the request relevant?" } },+ };+ const state = Array.from({ length: 160 }, (_, i) => `word${i}`).join(" ");+ const opts = { model: "english", maxLen: 64, headMaxLen: 16, returnDetails: true as const };+ const single = await router.decide(state, schema, opts);+ const batch = await router.decideBatch([state, state], schema, opts);+ expect(single.usage).toMatchObject({ input_tokens: 64, state_tokens_dropped: 114, truncated: true });+ expect(batch.map((result) => result.usage)).toEqual([single.usage, single.usage]);+ expect(batch.map((result) => result.values)).toEqual([single.values, single.values]);+ expect(encoderLengths).toEqual([[64], [64, 64]]);+ });+});+// `DecideBatchOptions` documents that anything it does not read itself is forwarded to the// runner, hooks included. On a Router-like runner the call used to spread those options into the// request objects and pass no third argument, so `Router.predictBatch` -- which reads hooks from