Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): port Agent.predict_batch and Router.route_batch/predict_batch

Aashish7689f2f4fa88
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 6a8c8566..d4635316 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -128,6 +128,11 @@ export interface PredictOptions {
onPredictStart?: PredictHook;
onPredictEnd?: PredictHook;
hooksRaise?: boolean;
+ /** Per-call token-budget overrides; null = agent config. A start hook may also set them. */
+ maxLen?: number | null;
+ headMaxLen?: number | null;
+ /** predictBatch: cap on states per shared forward pass; null sends them all in one pass. */
+ batchSize?: number | null;
}
function qidStr(qid: string): string {
@@ -380,20 +385,18 @@ export class Agent extends HookRegistry {
states,
questions: questions as Record<string, unknown>,
agent: this,
+ maxLen: opts.maxLen ?? null,
+ headMaxLen: opts.headMaxLen ?? null,
});
try {
await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors });
if (ctx.results === null) {
- const out: SystemOneResult[] = [];
- for (const st of ctx.states) {
- out.push(
- await this._systemOneCore(
- st,
- ctx.questions as Record<string, QuestionDef>,
- opts.lang ?? null,
- ),
- );
- }
+ const out = await this._systemOneMany(ctx.states, ctx.questions as Record<string, QuestionDef>, {
+ maxLen: ctx.maxLen,
+ headMaxLen: ctx.headMaxLen,
+ batchSize: opts.batchSize ?? null,
+ lang: opts.lang ?? null,
+ });
ctx.results = out as unknown as Record<string, unknown>[];
ctx.model ??= out[0]?.model ?? null;
}
@@ -418,116 +421,187 @@ export class Agent extends HookRegistry {
return ctx.results as unknown as SystemOneResult[];
}
- private async _systemOneCore(
+ /** Encode one state into one sequence row per question; the state text is tokenized once. */
+ private _encodeItems(
state: unknown,
- questions: Record<string, QuestionDef>,
- lang: string | null = null,
- ): Promise<SystemOneResult> {
- const ids = Object.keys(questions ?? {});
- if (ids.length === 0) {
- return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } };
- }
- const items: { ids: number[]; markers: number[]; qtype: number }[] = [];
- const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }[] = [];
+ ids: string[],
+ internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[],
+ maxLen: number,
+ headMaxLen: number,
+ ): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number } {
// The state text is shared by every question and is usually the longest text in the
// sequence — encode it once and compose the per-question prefixes onto it.
const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));
- for (const qid of ids) {
- checkQuestion(qid, questions[qid]);
- const q = toInternal(questions[qid]);
- internals.push(q);
- const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen);
+ const items: { ids: number[]; markers: number[]; qtype: number }[] = [];
+ for (let qi = 0; qi < ids.length; qi++) {
+ const qid = ids[qi];
+ const q = internals[qi];
+ const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen);
const { ids: seq, markers } = sequenceWithState(
- prefix, stAll, this.tok.sepId, this.maxLen, Array.isArray(state),
+ prefix,
+ stAll,
+ this.tok.sepId,
+ maxLen,
+ Array.isArray(state),
);
if (markers.length !== renderOptions(q).length) {
- throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`);
+ throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${headMaxLen}`);
}
items.push({ ids: seq, markers, qtype: QTYPES[q.t] });
}
- const collated = collateItems([items], this.tok.padId);
- if (!collated) throw new Error("no items to collate");
- const batch: Batch = collated;
- const nTokens = batch.attentionMask.flat().reduce((a, b) => a + b, 0);
- const { lastHidden } = await this.provider.runEncoder(batch);
- const { logits, act } = await this.provider.runHead(lastHidden, batch);
- if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < ids.length || act.length < ids.length) {
- throw new Error("model provider returned fewer output rows than input items");
+ return { items, nTokens: items.reduce((a, it) => a + it.ids.length, 0) };
+ }
+
+ /**
+ * Evaluate one shared question schema over many states. Each chunk of up to `batchSize`
+ * states is collated into a single encoder+head forward pass; results align with `states`
+ * by index. (Python `Agent.predict_batch` parity; its `sort_by_length` is not ported.)
+ */
+ private async _systemOneMany(
+ states: unknown[],
+ questions: Record<string, QuestionDef>,
+ opts: {
+ maxLen?: number | null;
+ headMaxLen?: number | null;
+ batchSize?: number | null;
+ lang?: string | null;
+ } = {},
+ ): Promise<SystemOneResult[]> {
+ if (states.length === 0) return [];
+ const ids = Object.keys(questions ?? {});
+ if (ids.length === 0) {
+ return states.map(() => ({
+ model: "laya-rl-agent",
+ answers: {},
+ usage: { input_tokens: 0, output_tokens: 0 },
+ }));
}
- for (let r = 0; r < ids.length; r++) {
- const k = items[r].markers.length;
- if (!Array.isArray(logits[r]) || logits[r].length < k || !Array.isArray(act[r]) || act[r].length === 0 ||
- logits[r].some((v) => !Number.isFinite(v)) || act[r].some((v) => !Number.isFinite(v))) {
- throw new Error(`model provider returned invalid output for question ${ids[r]}`);
+ const maxLen = opts.maxLen ?? this.maxLen;
+ const headMaxLen = opts.headMaxLen ?? this.headMaxLen;
+ const internals = ids.map((qid) => {
+ checkQuestion(qid, questions[qid]);
+ return toInternal(questions[qid]);
+ });
+ const chunkSize =
+ opts.batchSize && opts.batchSize > 0 ? Math.floor(opts.batchSize) : states.length;
+ const out: SystemOneResult[] = [];
+ for (let start = 0; start < states.length; start += chunkSize) {
+ const chunk = states.slice(start, start + chunkSize);
+ const built = chunk.map((st) => this._encodeItems(st, ids, internals, maxLen, headMaxLen));
+ const collated = collateItems(built.map((b) => b.items), this.tok.padId);
+ if (!collated) throw new Error("no items to collate");
+ const batch: Batch = collated;
+ const { lastHidden } = await this.provider.runEncoder(batch);
+ const { logits, act } = await this.provider.runHead(lastHidden, batch);
+ const rows = chunk.length * ids.length;
+ if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < rows || act.length < rows) {
+ throw new Error("model provider returned fewer output rows than input items");
+ }
+ for (let s = 0; s < chunk.length; s++) {
+ for (let qi = 0; qi < ids.length; qi++) {
+ const r = s * ids.length + qi;
+ const k = built[s].items[qi].markers.length;
+ const logitRow = logits[r] as number[];
+ const actRow = act[r] as number[];
+ if (
+ !Array.isArray(logitRow) || logitRow.length < k ||
+ !Array.isArray(actRow) || actRow.length === 0 ||
+ logitRow.some((v) => !Number.isFinite(v)) || actRow.some((v) => !Number.isFinite(v))
+ ) {
+ throw new Error(`model provider returned invalid output for question ${ids[qi]}`);
+ }
+ }
}
- }
- const answers: Record<string, SystemAnswer> = {};
- for (let r = 0; r < ids.length; r++) {
- const qid = ids[r];
- const q = internals[r];
- const k = items[r].markers.length;
- const qt = QTYPES[q.t];
- const bucket = tempBucket(qt, k);
- // Python parity (_decode_answers): a matching lang override replaces the scale
- // wholesale — its own temperature_by_options first, then its 3-slot temperature —
- // so an override without buckets intentionally ignores the base per-bucket entries.
- const langCfg = lang ? this.langTemperatures[lang.split("-")[0].toLowerCase()] : undefined;
- const scale = langCfg
- ? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0)
- : (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0);
- const z = (logits[r] as number[]).slice(0, k).map((v) => v / scale);
- const p = softmax(z);
- const actRow = (act[r] as number[]) ?? [1, 0];
- const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
- const ext = { act_probability: r4(actP[0]) };
- // Same quantity on every question type (max(p)), so callers can gate across types on
- // one number; `confidence` stays as-is for existing callers (entropy for choice/score).
- const ansConf = r4(answerConfidence(p));
- if (q.t === "choice") {
- const keys = Object.keys(q.crit as Record<string, unknown>);
- let best = 0;
- for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i;
- answers[qid] = {
- type: "choice",
- choice: keys[best],
- probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),
- confidence: r4(confidenceFromProbs(p)),
- answer_confidence: ansConf,
- action: ext,
- };
- } else if (q.t === "score") {
- const exp = p.reduce((a, v, i) => a + i * v, 0);
- answers[qid] = {
- type: "score",
- score: r4(exp),
- // `renderCriterion`, not the raw `c`: a legend maps an index to the TEXT of a level, and
- // the keys are already strings. A numeric scale written directly used to come back with
- // the caller's own JSON types (`{"0": 1}`), so a client that reads a level as a string
- // had to handle a number, a boolean and null as well -- and `{"0": null}` is not even
- // parseable by a Jev client (#302). Same change as the Python `agent.py` and
- // `onnx_agent.py` legends, so the two runtimes return the same value types.
- legend: Object.fromEntries(
- (q.crit as unknown[]).map((c, i) => [String(i), renderCriterion(c)]),
- ),
- probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),
- confidence: r4(confidenceFromProbs(p)),
- answer_confidence: ansConf,
- action: ext,
- };
- } else {
- const pt = p[1] ?? 0;
- answers[qid] = {
- type: "noul",
- noul: r4(pt),
- confidence: r4(Math.max(pt, 1 - pt)),
- // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here
- answer_confidence: ansConf,
- action: ext,
- };
+ let row = 0;
+ for (let s = 0; s < chunk.length; s++) {
+ const answers: Record<string, SystemAnswer> = {};
+ for (let qi = 0; qi < ids.length; qi++) {
+ answers[ids[qi]] = this._decodeRow(
+ logits[row] as number[],
+ (act[row] as number[]) ?? [1, 0],
+ internals[qi],
+ built[s].items[qi].markers.length,
+ opts.lang ?? null,
+ );
+ row++;
+ }
+ out.push({
+ model: "laya-rl-agent",
+ answers,
+ usage: { input_tokens: built[s].nTokens, output_tokens: 0 },
+ });
}
}
- return { model: "laya-rl-agent", answers, usage: { input_tokens: nTokens, output_tokens: 0 } };
+ return out;
+ }
+
+ /** Decode one head row into the answer for its question. */
+ private _decodeRow(
+ logitsRow: number[],
+ actRow: number[],
+ q: { t: "choice" | "score" | "noul"; ins: string; crit: unknown },
+ k: number,
+ lang: string | null = null,
+ ): SystemAnswer {
+ const qt = QTYPES[q.t];
+ const bucket = tempBucket(qt, k);
+ // Python parity (_decode_answers): a matching lang override replaces the scale
+ // wholesale — its own temperature_by_options first, then its 3-slot temperature —
+ // so an override without buckets intentionally ignores the base per-bucket entries.
+ const langCfg = lang ? this.langTemperatures[lang.split("-")[0].toLowerCase()] : undefined;
+ const scale = langCfg
+ ? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0)
+ : (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0);
+ const z = logitsRow.slice(0, k).map((v) => v / scale);
+ const p = softmax(z);
+ const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));
+ const ext = { act_probability: r4(actP[0]) };
+ // Same quantity on every question type (max(p)), so callers can gate across types on
+ // one number; `confidence` stays as-is for existing callers (entropy for choice/score).
+ const ansConf = r4(answerConfidence(p));
+ if (q.t === "choice") {
+ const keys = Object.keys(q.crit as Record<string, unknown>);
+ let best = 0;
+ for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i;
+ return {
+ type: "choice",
+ choice: keys[best],
+ probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),
+ confidence: r4(confidenceFromProbs(p)),
+ answer_confidence: ansConf,
+ action: ext,
+ };
+ }
+ if (q.t === "score") {
+ const exp = p.reduce((a, v, i) => a + i * v, 0);
+ return {
+ type: "score",
+ score: r4(exp),
+ // `renderCriterion`, not the raw `c`: a legend maps an index to the TEXT of a level, and
+ // the keys are already strings. A numeric scale written directly used to come back with
+ // the caller's own JSON types (`{"0": 1}`), so a client that reads a level as a string
+ // had to handle a number, a boolean and null as well -- and `{"0": null}` is not even
+ // parseable by a Jev client (#302). Same change as the Python `agent.py` and
+ // `onnx_agent.py` legends, so the two runtimes return the same value types.
+ legend: Object.fromEntries(
+ (q.crit as unknown[]).map((c, i) => [String(i), renderCriterion(c)]),
+ ),
+ probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),
+ confidence: r4(confidenceFromProbs(p)),
+ answer_confidence: ansConf,
+ action: ext,
+ };
+ }
+ const pt = p[1] ?? 0;
+ return {
+ type: "noul",
+ noul: r4(pt),
+ confidence: r4(Math.max(pt, 1 - pt)),
+ // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here
+ answer_confidence: ansConf,
+ action: ext,
+ };
}
async predict(
@@ -537,6 +611,25 @@ export class Agent extends HookRegistry {
): Promise<SystemOneResult> {
return this.systemOne(state, questions, opts);
}
+ /**
+ * Evaluate the same questions over many states, packing each chunk of up to
+ * `opts.batchSize` states into one shared forward pass (Python `Agent.predict_batch`
+ * parity). This is the throughput path: `systemOne`/`predict` run one state per forward
+ * pass, which leaves most of the batch dimension idle on GPU. Results align with
+ * `states` by index and are identical in shape to `systemOne`'s output.
+ *
+ * `opts.maxLen` / `opts.headMaxLen` override the agent config for this call; a start
+ * hook may also set `ctx.maxLen` / `ctx.headMaxLen` or call `ctx.skip(results)`.
+ * Python's `sort_by_length` grouping is not ported.
+ */
+ async predictBatch(
+ states: unknown[],
+ questions: Record<string, QuestionDef>,
+ opts: PredictOptions = {},
+ ): Promise<SystemOneResult[]> {
+ return this._predictHooked(states, questions, opts);
+ }
+
/**
* Answer `state` against a JSON schema (or explicit `opts.questions`) and return typed
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index 38e105a3..b311d389 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -16,7 +16,7 @@ export type {
export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle, loadWebBundle, PINNED_REVISIONS, resolveRevision } from "./providers.js";
export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js";
export { Router, normaliseName, DEFAULT_MODELS } from "./router.js";
-export type { RoutedResult, RouteDecision, ModelName, ModelSpec } from "./router.js";
+export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest } from "./router.js";
export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K } from "./shortlist.js";
export type { EmbedFn, ShortlistMeta } from "./shortlist.js";
export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js";
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index 33ded388..fc968c0b 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -10,6 +10,7 @@ import {
markDefaultsRan,
dispatchAsync,
normaliseHooks,
+ type Hook,
type HookArg,
type PredictHook,
} from "./hooks.js";
@@ -148,6 +149,30 @@ export interface RouteOptions {
hooksRaise?: boolean;
}
+/** One item of a Router.predictBatch/routeBatch batch: state, questions, route overrides. */
+export interface BatchRequest {
+ state: unknown;
+ questions: Record<string, QuestionDef>;
+ model?: string | null;
+ task?: string | null;
+ lang?: string | null;
+ langGuess?: LangGuess;
+ lang_guess?: LangGuess;
+}
+
+function questionSchema(questions: Record<string, unknown>): string {
+ // Python groups on json.dumps(sort_keys=False, default=str): insertion order is significant
+ // at every nesting level because options are positional in the rendered sequence, so two
+ // equal schemas with different key orders must not share a forward pass. JSON.stringify
+ // preserves insertion order for string keys; the replacer stands in for default=str on
+ // the few value types JSON cannot represent.
+ return JSON.stringify(questions, (_key, value) =>
+ typeof value === "function" || typeof value === "symbol" || typeof value === "bigint"
+ ? String(value)
+ : value,
+ );
+}
+
function toSpec(spec: string | ModelSpec | [string, string | null]): ModelSpec {
if (typeof spec === "string") return { repo: spec, subfolder: null };
if (Array.isArray(spec)) {
@@ -579,4 +604,238 @@ export class Router extends HookRegistry {
): Promise<RoutedResult> {
return this.predict(state, questions, opts);
}
+
+ /**
+ * Route many requests in one call (Python `Router.route_batch` parity). Every request is
+ * validated before anything loads, so a malformed batch fails fast. Entries keep input
+ * order; routing is deterministic, so repeated batches route identically.
+ */
+ routeBatch(requests: BatchRequest[]): RouteDecision[] {
+ if (!Array.isArray(requests)) {
+ throw new TypeError("requests must be an array of request objects");
+ }
+ const decisions: RouteDecision[] = [];
+ for (let i = 0; i < requests.length; i++) {
+ const request = requests[i];
+ if (typeof request !== "object" || request === null || Array.isArray(request)) {
+ const got = Array.isArray(request) ? "list" : request === null ? "null" : typeof request;
+ throw new TypeError(`request ${i} must be an object, got ${got}`);
+ }
+ if (!("state" in request)) {
+ throw new Error(`request ${i} is missing required key 'state'`);
+ }
+ if (!("questions" in request)) {
+ throw new Error(`request ${i} is missing required key 'questions'`);
+ }
+ const questions = request.questions;
+ if (typeof questions !== "object" || questions === null || Array.isArray(questions)) {
+ const got =
+ Array.isArray(questions) ? "list" : questions === null ? "null" : typeof questions;
+ throw new TypeError(`request ${i} 'questions' must be an object, got ${got}`);
+ }
+ decisions.push(
+ this.route(request.state, questions, {
+ model: request.model ?? null,
+ task: request.task ?? null,
+ lang: request.lang ?? null,
+ langGuess: request.langGuess ?? request.lang_guess ?? null,
+ }),
+ );
+ }
+ return decisions;
+ }
+
+ /**
+ * Route and run many requests in one call (Python `Router.predict_batch` parity).
+ * Requests are routed first, then grouped by checkpoint so each loaded Agent scores its
+ * requests in as few forward passes as possible; results are restored to input order.
+ * Requests routed to the same checkpoint still split into separate `predictBatch` calls
+ * when their question schemas differ (order-sensitively) or when per-request start hooks
+ * set different maxLen/headMaxLen overrides.
+ *
+ * Router-level predict hooks run per request: each request gets its own PredictContext
+ * carrying `decision`; `onPredictStart` may rewrite a request or `ctx.skip()` it, and
+ * `onPredictEnd` runs once per started request even when the batch fails.
+ */
+ async predictBatch(
+ requests: BatchRequest[],
+ batchSize: number | null = null,
+ ): Promise<RoutedResult[]> {
+ const decisions = this.routeBatch(requests);
+ if (decisions.length === 0) return [];
+
+ const groups = new Map<string, number[]>();
+ for (let i = 0; i < decisions.length; i++) {
+ const model = decisions[i].model;
+ const bucket = groups.get(model);
+ if (bucket) bucket.push(i);
+ else groups.set(model, [i]);
+ }
+
+ const results: (RoutedResult | null)[] = new Array(requests.length).fill(null);
+ const active = composeHooks(this.hooks);
+ const raiseErrors = this.hooksRaise;
+
+ for (const [modelName, indices] of groups) {
+ const agent = (await this.load(modelName)) as {
+ predictBatch?(
+ states: unknown[],
+ questions: Record<string, QuestionDef>,
+ opts?: { batchSize?: number | null; maxLen?: number | null; headMaxLen?: number | null },
+ ): Promise<SystemOneResult[]>;
+ systemOne(
+ state: unknown,
+ questions: Record<string, QuestionDef>,
+ ): Promise<SystemOneResult>;
+ };
+ const started: PredictContext[] = [];
+ try {
+ for (const i of indices) {
+ const ctx = new PredictContext({
+ states: [requests[i].state],
+ questions: requests[i].questions as Record<string, unknown>,
+ decision: { ...decisions[i] } as unknown as Record<string, unknown>,
+ model: modelName,
+ agent,
+ router: this,
+ });
+ started.push(ctx);
+ dispatch(active, "onPredictStart", ctx, { raiseErrors });
+ }
+
+ // Order-sensitive at every nesting level (#166): options are positional, so two
+ // equal schemas with different key orders must not share a group.
+ const questionGroups: {
+ questions: Record<string, unknown>;
+ schema: string;
+ maxLen: number | null;
+ headMaxLen: number | null;
+ items: [number, PredictContext][];
+ }[] = [];
+ for (let s = 0; s < indices.length; s++) {
+ const i = indices[s];
+ const ctx = started[s];
+ if (ctx.results !== null) {
+ // A cache hit short-circuits inference, but predict still promises a `routing`
+ // key. Add it without overwriting a routing the cached payload already has.
+ for (const result of ctx.results) {
+ if (result && typeof result === "object" && !("routing" in result)) {
+ (result as unknown as RoutedResult).routing = { ...decisions[i] };
+ }
+ }
+ continue;
+ }
+ const schema = questionSchema(ctx.questions);
+ const found = questionGroups.find(
+ (g) => g.schema === schema && g.maxLen === ctx.maxLen && g.headMaxLen === ctx.headMaxLen,
+ );
+ if (found) found.items.push([i, ctx]);
+ else {
+ questionGroups.push({
+ questions: ctx.questions,
+ schema,
+ maxLen: ctx.maxLen,
+ headMaxLen: ctx.headMaxLen,
+ items: [[i, ctx]],
+ });
+ }
+ }
+
+ for (const group of questionGroups) {
+ const agentOpts: {
+ batchSize: number | null;
+ maxLen?: number;
+ headMaxLen?: number;
+ } = { batchSize };
+ // Only pass overrides when set, so an Agent-like object that does not accept
+ // them still works.
+ if (group.maxLen !== null) agentOpts.maxLen = group.maxLen;
+ if (group.headMaxLen !== null) agentOpts.headMaxLen = group.headMaxLen;
+ let batchResults: SystemOneResult[];
+ if (typeof agent.predictBatch === "function") {
+ batchResults = await agent.predictBatch(
+ group.items.map(([, ctx]) => ctx.states[0]),
+ group.questions as Record<string, QuestionDef>,
+ agentOpts,
+ );
+ } else {
+ // Agent-like objects that only implement systemOne (light wrappers, tests)
+ // still work; their states simply share no forward pass.
+ batchResults = [];
+ for (const [, ctx] of group.items) {
+ batchResults.push(
+ await agent.systemOne(ctx.states[0], group.questions as Record<string, QuestionDef>),
+ );
+ }
+ }
+ if (batchResults.length !== group.items.length) {
+ throw new Error(
+ `internal error: Agent.predictBatch returned ${batchResults.length} results for ${group.items.length} states`,
+ );
+ }
+ for (let s = 0; s < group.items.length; s++) {
+ const [i, ctx] = group.items[s];
+ const result = batchResults[s] as RoutedResult;
+ result.routing = { ...decisions[i] };
+ ctx.results = [result as unknown as Record<string, unknown>];
+ }
+ }
+ } catch (err) {
+ for (const ctx of started) {
+ if (ctx.results === null) {
+ ctx.error = err;
+ try {
+ dispatch(active, "onError", ctx, { raiseErrors });
+ } catch {
+ // Chained onto the batch failure, which is the one that propagates.
+ }
+ }
+ }
+ try {
+ this._endContexts(active, started, raiseErrors);
+ } catch {
+ // A failing end hook is chained onto the batch failure, not raised instead.
+ }
+ throw err;
+ }
+ this._endContexts(active, started, raiseErrors);
+ for (let s = 0; s < indices.length; s++) {
+ results[indices[s]] = started[s].results![0] as unknown as RoutedResult;
+ }
+ }
+
+ if (results.some((r) => r === null)) {
+ throw new Error("internal error: batch execution did not produce every result");
+ }
+ return results as RoutedResult[];
+ }
+
+ /** Alias of predictBatch, mirroring Python's `Router.predict_many`. */
+ async predictMany(
+ requests: BatchRequest[],
+ batchSize: number | null = null,
+ ): Promise<RoutedResult[]> {
+ return this.predictBatch(requests, batchSize);
+ }
+
+ /**
+ * Run onPredictEnd for every started per-request context and raise the first hook
+ * failure, if any. Elapsed and usage are stamped on every context before any end hook
+ * runs, so one request's end hooks never inflate another request's elapsed time.
+ */
+ private _endContexts(active: Hook[], contexts: PredictContext[], raiseErrors: boolean): void {
+ for (const ctx of contexts) {
+ ctx.markElapsed();
+ if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);
+ }
+ let firstError: unknown = null;
+ for (const ctx of contexts) {
+ try {
+ dispatch(active, "onPredictEnd", ctx, { raiseErrors });
+ } catch (hookErr) {
+ if (ctx.error === null && firstError === null) firstError = hookErr;
+ }
+ }
+ if (firstError !== null) throw firstError;
+ }
}