Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): forward each request's lang on the batch path (#381 parity)

Aashishf1e2ac030aad
diff --git a/laya-ts/README.md b/laya-ts/README.md
index f08e20f4..5e4a6148 100644
--- a/laya-ts/README.md
+++ b/laya-ts/README.md
@@ -114,10 +114,13 @@ const decisions = router.routeBatch(requests); // validate + route, not
const routed = await router.predictBatch(requests, 32); // == router.predictMany(...)
// requests routed to the same checkpoint still split into separate batches when their
-// question schemas differ (order-sensitively) or when per-request start hooks set
-// different ctx.maxLen / ctx.headMaxLen overrides. Router-level predict hooks run once
-// per request: ctx.decision is set, ctx.skip() serves a cached result, and onPredictEnd
-// runs per request even when the batch fails.
+// question schemas differ (order-sensitively), when per-request start hooks set
+// different ctx.maxLen / ctx.headMaxLen overrides, or — for an agent carrying
+// lang_temperatures — when their languages differ: each request's effective language (an
+// explicit lang, otherwise the detected non-English one) is forwarded to the agent, so
+// the batched path scores exactly like predict. Router-level predict hooks run once per
+// request: ctx.decision is set, ctx.skip() serves a cached result, and onPredictEnd runs
+// per request even when the batch fails.
```
Python's `sort_by_length` grouping is not ported yet.
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index fc968c0b..615436de 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -650,8 +650,11 @@ export class Router extends HookRegistry {
* Requests are routed first, then grouped by checkpoint so each loaded Agent scores its
* requests in as few forward passes as possible; results are restored to input order.
* Requests routed to the same checkpoint still split into separate `predictBatch` calls
- * when their question schemas differ (order-sensitively) or when per-request start hooks
- * set different maxLen/headMaxLen overrides.
+ * when their question schemas differ (order-sensitively), when per-request start hooks
+ * set different maxLen/headMaxLen overrides, or — for an agent carrying
+ * `lang_temperatures` — when their languages differ: each request's effective language
+ * (an explicit `lang`, otherwise the detected non-English one) is forwarded so the
+ * batched path scores exactly like `predict`.
*
* Router-level predict hooks run per request: each request gets its own PredictContext
* carrying `decision`; `onPredictStart` may rewrite a request or `ctx.skip()` it, and
@@ -681,12 +684,19 @@ export class Router extends HookRegistry {
predictBatch?(
states: unknown[],
questions: Record<string, QuestionDef>,
- opts?: { batchSize?: number | null; maxLen?: number | null; headMaxLen?: number | null },
+ opts?: {
+ batchSize?: number | null;
+ maxLen?: number | null;
+ headMaxLen?: number | null;
+ lang?: string | null;
+ },
): Promise<SystemOneResult[]>;
systemOne(
state: unknown,
questions: Record<string, QuestionDef>,
+ opts?: { lang?: string | null },
): Promise<SystemOneResult>;
+ langTemperatures?: Record<string, unknown>;
};
const started: PredictContext[] = [];
try {
@@ -705,11 +715,20 @@ export class Router extends HookRegistry {
// Order-sensitive at every nesting level (#166): options are positional, so two
// equal schemas with different key orders must not share a group.
+ //
+ // Python parity (router.py predict_batch): `predict` forwards the request's language so
+ // the agent can apply its per-language temperatures; the batched path forwarded only the
+ // token budgets, so the same request scored differently depending on the entry point.
+ // Only computed for an agent that actually carries overrides: `lang` is otherwise
+ // unused, and adding it to the group key would split a group that shares one forward
+ // pass today.
+ const hasLangOverrides = Object.keys(agent.langTemperatures ?? {}).length > 0;
const questionGroups: {
questions: Record<string, unknown>;
schema: string;
maxLen: number | null;
headMaxLen: number | null;
+ lang: string | null;
items: [number, PredictContext][];
}[] = [];
for (let s = 0; s < indices.length; s++) {
@@ -725,9 +744,21 @@ export class Router extends HookRegistry {
}
continue;
}
+ // An explicit `lang` wins; otherwise forward the language the router detected for the
+ // routing decision. TS analyse() names English "en" where Python's returns None (it
+ // only ever names non-English), so a detected "en" forwards as null — in Python only
+ // an explicit lang="en" can select an "en" override. Same rule as `predict`.
+ const detectedLang = decisions[i].detection?.language ?? null;
+ const effectiveLang =
+ requests[i].lang ?? (detectedLang && detectedLang !== "en" ? detectedLang : null);
+ const langKey = hasLangOverrides ? effectiveLang : null;
const schema = questionSchema(ctx.questions);
const found = questionGroups.find(
- (g) => g.schema === schema && g.maxLen === ctx.maxLen && g.headMaxLen === ctx.headMaxLen,
+ (g) =>
+ g.schema === schema &&
+ g.maxLen === ctx.maxLen &&
+ g.headMaxLen === ctx.headMaxLen &&
+ g.lang === langKey,
);
if (found) found.items.push([i, ctx]);
else {
@@ -736,6 +767,7 @@ export class Router extends HookRegistry {
schema,
maxLen: ctx.maxLen,
headMaxLen: ctx.headMaxLen,
+ lang: langKey,
items: [[i, ctx]],
});
}
@@ -746,11 +778,13 @@ export class Router extends HookRegistry {
batchSize: number | null;
maxLen?: number;
headMaxLen?: number;
+ lang?: string;
} = { batchSize };
// Only pass overrides when set, so an Agent-like object that does not accept
// them still works.
if (group.maxLen !== null) agentOpts.maxLen = group.maxLen;
if (group.headMaxLen !== null) agentOpts.headMaxLen = group.headMaxLen;
+ if (group.lang !== null) agentOpts.lang = group.lang;
let batchResults: SystemOneResult[];
if (typeof agent.predictBatch === "function") {
batchResults = await agent.predictBatch(
@@ -764,7 +798,11 @@ export class Router extends HookRegistry {
batchResults = [];
for (const [, ctx] of group.items) {
batchResults.push(
- await agent.systemOne(ctx.states[0], group.questions as Record<string, QuestionDef>),
+ await agent.systemOne(
+ ctx.states[0],
+ group.questions as Record<string, QuestionDef>,
+ group.lang === null ? undefined : { lang: group.lang },
+ ),
);
}
}
diff --git a/laya-ts/tests/lang-temperature.test.ts b/laya-ts/tests/lang-temperature.test.ts
index 890a335f..c42f90ef 100644
--- a/laya-ts/tests/lang-temperature.test.ts
+++ b/laya-ts/tests/lang-temperature.test.ts
@@ -136,3 +136,86 @@ describe("Router.predict lang forwarding", () => {
expect(seen).toEqual([{ lang: "en" }]);
});
});
+
+describe("Router.predictBatch lang forwarding", () => {
+ const questions = { q: { type: "noul", instructions: "?" } } as any;
+ const req = (state: unknown, overrides: Record<string, unknown> = {}) =>
+ ({ state, questions, ...overrides }) as any;
+
+ // Records every agent.predictBatch call. `langTemperatures` is what gates the group split,
+ // exactly as on Agent: an agent without overrides keeps sharing one forward pass.
+ function routerRecording(seen: any[], langTemperatures: Record<string, unknown> = { de: {} }) {
+ const stub = {
+ langTemperatures,
+ async predictBatch(states: unknown[], _q: any, opts?: any) {
+ seen.push({ states, opts });
+ return states.map((state) => ({ model: "m", answers: { seen: state }, usage: {} }));
+ },
+ async systemOne(state: unknown, _q: any, opts?: any) {
+ seen.push({ states: [state], opts });
+ return { model: "m", answers: { seen: state }, usage: {} };
+ },
+ };
+ return new Router({ loader: () => stub } as any);
+ }
+
+ it("forwards an explicit lang to agent.predictBatch, winning over detection", async () => {
+ const seen: any[] = [];
+ const r = routerRecording(seen);
+ // This romanized Bangla text detects as "bn"; the explicit lang must win.
+ await r.predictBatch([req("ami ekta ticket khulsi, kalke theke payment hocche na", { lang: "de" })]);
+ expect(seen.map((c) => c.opts.lang)).toEqual(["de"]);
+ });
+
+ it("forwards the detected language when no explicit lang is given", async () => {
+ const seen: any[] = [];
+ const r = routerRecording(seen);
+ await r.predictBatch([req("ami ekta ticket khulsi, kalke theke payment hocche na")]);
+ expect(seen.map((c) => c.opts.lang)).toEqual(["bn"]);
+ });
+
+ it("forwards no lang when nothing non-English was detected", async () => {
+ const seen: any[] = [];
+ const r = routerRecording(seen);
+ await r.predictBatch([req("please refund my ticket")]);
+ expect(seen.map((c) => c.opts.lang)).toEqual([undefined]);
+ });
+
+ it("does not share a forward pass across languages", async () => {
+ const seen: any[] = [];
+ const r = routerRecording(seen);
+ await r.predictBatch([
+ req("hallo", { lang: "de", model: "english" }),
+ req("buna", { lang: "ro", model: "english" }),
+ req("hallo zwei", { lang: "de", model: "english" }),
+ ]);
+ expect(seen.map((c) => [c.opts.lang, c.states])).toEqual([
+ ["de", ["hallo", "hallo zwei"]],
+ ["ro", ["buna"]],
+ ]);
+ });
+
+ it("keeps one shared batch when the agent carries no lang overrides", async () => {
+ const seen: any[] = [];
+ const r = routerRecording(seen, {});
+ await r.predictBatch([
+ req("hallo", { lang: "de", model: "english" }),
+ req("hello", { lang: "fr", model: "english" }),
+ ]);
+ expect(seen.map((c) => [c.opts.lang, c.states])).toEqual([[undefined, ["hallo", "hello"]]]);
+ });
+
+ it("applies the same lang override on the batch path as predict (German override)", async () => {
+ // The reported gap: with a German override, predict returned 0.7311 and predictBatch
+ // 0.8808, because the batched path never forwarded the request's language.
+ const agent = new Agent({
+ provider: fakeProvider(),
+ lang_temperatures: { de: { temperature: [2, 2, 2] } },
+ } as any);
+ const r = new Router({ loader: () => agent } as any);
+ const single: any = await r.predict("hallo", CHOICE, { lang: "de" });
+ const batch: any[] = await r.predictBatch([{ state: "hallo", questions: CHOICE, lang: "de" } as any]);
+ expect(single.answers.d.probabilities.x).toBeCloseTo(0.7311, 4);
+ expect(batch[0].answers.d.probabilities.x).toBeCloseTo(0.7311, 4);
+ });
+});