Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(ts): port Agent.predict_batch and Router.route_batch/predict_batch
Aashish7689f2f4fa88
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.tsindex 6a8c8566..d4635316 100644--- a/laya-ts/src/agent.ts+++ b/laya-ts/src/agent.ts@@ -128,6 +128,11 @@ export interface PredictOptions {onPredictStart?: PredictHook;onPredictEnd?: PredictHook;hooksRaise?: boolean;+ /** Per-call token-budget overrides; null = agent config. A start hook may also set them. */+ maxLen?: number | null;+ headMaxLen?: number | null;+ /** predictBatch: cap on states per shared forward pass; null sends them all in one pass. */+ batchSize?: number | null;}function qidStr(qid: string): string {@@ -380,20 +385,18 @@ export class Agent extends HookRegistry {states,questions: questions as Record<string, unknown>,agent: this,+ maxLen: opts.maxLen ?? null,+ headMaxLen: opts.headMaxLen ?? null,});try {await dispatchAsync(active, "onPredictStart", ctx, { raiseErrors });if (ctx.results === null) {- const out: SystemOneResult[] = [];- for (const st of ctx.states) {- out.push(- await this._systemOneCore(- st,- ctx.questions as Record<string, QuestionDef>,- opts.lang ?? null,- ),- );- }+ const out = await this._systemOneMany(ctx.states, ctx.questions as Record<string, QuestionDef>, {+ maxLen: ctx.maxLen,+ headMaxLen: ctx.headMaxLen,+ batchSize: opts.batchSize ?? null,+ lang: opts.lang ?? null,+ });ctx.results = out as unknown as Record<string, unknown>[];ctx.model ??= out[0]?.model ?? null;}@@ -418,116 +421,187 @@ export class Agent extends HookRegistry {return ctx.results as unknown as SystemOneResult[];}- private async _systemOneCore(+ /** Encode one state into one sequence row per question; the state text is tokenized once. */+ private _encodeItems(state: unknown,- questions: Record<string, QuestionDef>,- lang: string | null = null,- ): Promise<SystemOneResult> {- const ids = Object.keys(questions ?? {});- if (ids.length === 0) {- return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } };- }- const items: { ids: number[]; markers: number[]; qtype: number }[] = [];- const internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown; labels?: { false: string; true: string } }[] = [];+ ids: string[],+ internals: { t: "choice" | "score" | "noul"; ins: string; crit: unknown }[],+ maxLen: number,+ headMaxLen: number,+ ): { items: { ids: number[]; markers: number[]; qtype: number }[]; nTokens: number } {// The state text is shared by every question and is usually the longest text in the// sequence — encode it once and compose the per-question prefixes onto it.const stAll = this.tok.encode(serializeState(state).split(this.tok.maskToken).join(" "));- for (const qid of ids) {- checkQuestion(qid, questions[qid]);- const q = toInternal(questions[qid]);- internals.push(q);- const prefix = buildQuestionPrefix(this.tok, q, this.maxLen, this.headMaxLen);+ const items: { ids: number[]; markers: number[]; qtype: number }[] = [];+ for (let qi = 0; qi < ids.length; qi++) {+ const qid = ids[qi];+ const q = internals[qi];+ const prefix = buildQuestionPrefix(this.tok, q, maxLen, headMaxLen);const { ids: seq, markers } = sequenceWithState(- prefix, stAll, this.tok.sepId, this.maxLen, Array.isArray(state),+ prefix,+ stAll,+ this.tok.sepId,+ maxLen,+ Array.isArray(state),);if (markers.length !== renderOptions(q).length) {- throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${this.headMaxLen}`);+ throw new Error(`question ${qidStr(qid)} options exceed head_max_len=${headMaxLen}`);}items.push({ ids: seq, markers, qtype: QTYPES[q.t] });}- const collated = collateItems([items], this.tok.padId);- if (!collated) throw new Error("no items to collate");- const batch: Batch = collated;- const nTokens = batch.attentionMask.flat().reduce((a, b) => a + b, 0);- const { lastHidden } = await this.provider.runEncoder(batch);- const { logits, act } = await this.provider.runHead(lastHidden, batch);- if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < ids.length || act.length < ids.length) {- throw new Error("model provider returned fewer output rows than input items");+ return { items, nTokens: items.reduce((a, it) => a + it.ids.length, 0) };+ }++ /**+ * Evaluate one shared question schema over many states. Each chunk of up to `batchSize`+ * states is collated into a single encoder+head forward pass; results align with `states`+ * by index. (Python `Agent.predict_batch` parity; its `sort_by_length` is not ported.)+ */+ private async _systemOneMany(+ states: unknown[],+ questions: Record<string, QuestionDef>,+ opts: {+ maxLen?: number | null;+ headMaxLen?: number | null;+ batchSize?: number | null;+ lang?: string | null;+ } = {},+ ): Promise<SystemOneResult[]> {+ if (states.length === 0) return [];+ const ids = Object.keys(questions ?? {});+ if (ids.length === 0) {+ return states.map(() => ({+ model: "laya-rl-agent",+ answers: {},+ usage: { input_tokens: 0, output_tokens: 0 },+ }));}- for (let r = 0; r < ids.length; r++) {- const k = items[r].markers.length;- if (!Array.isArray(logits[r]) || logits[r].length < k || !Array.isArray(act[r]) || act[r].length === 0 ||- logits[r].some((v) => !Number.isFinite(v)) || act[r].some((v) => !Number.isFinite(v))) {- throw new Error(`model provider returned invalid output for question ${ids[r]}`);+ const maxLen = opts.maxLen ?? this.maxLen;+ const headMaxLen = opts.headMaxLen ?? this.headMaxLen;+ const internals = ids.map((qid) => {+ checkQuestion(qid, questions[qid]);+ return toInternal(questions[qid]);+ });+ const chunkSize =+ opts.batchSize && opts.batchSize > 0 ? Math.floor(opts.batchSize) : states.length;+ const out: SystemOneResult[] = [];+ for (let start = 0; start < states.length; start += chunkSize) {+ const chunk = states.slice(start, start + chunkSize);+ const built = chunk.map((st) => this._encodeItems(st, ids, internals, maxLen, headMaxLen));+ const collated = collateItems(built.map((b) => b.items), this.tok.padId);+ if (!collated) throw new Error("no items to collate");+ const batch: Batch = collated;+ const { lastHidden } = await this.provider.runEncoder(batch);+ const { logits, act } = await this.provider.runHead(lastHidden, batch);+ const rows = chunk.length * ids.length;+ if (!Array.isArray(logits) || !Array.isArray(act) || logits.length < rows || act.length < rows) {+ throw new Error("model provider returned fewer output rows than input items");+ }+ for (let s = 0; s < chunk.length; s++) {+ for (let qi = 0; qi < ids.length; qi++) {+ const r = s * ids.length + qi;+ const k = built[s].items[qi].markers.length;+ const logitRow = logits[r] as number[];+ const actRow = act[r] as number[];+ if (+ !Array.isArray(logitRow) || logitRow.length < k ||+ !Array.isArray(actRow) || actRow.length === 0 ||+ logitRow.some((v) => !Number.isFinite(v)) || actRow.some((v) => !Number.isFinite(v))+ ) {+ throw new Error(`model provider returned invalid output for question ${ids[qi]}`);+ }+ }}- }- const answers: Record<string, SystemAnswer> = {};- for (let r = 0; r < ids.length; r++) {- const qid = ids[r];- const q = internals[r];- const k = items[r].markers.length;- const qt = QTYPES[q.t];- const bucket = tempBucket(qt, k);- // Python parity (_decode_answers): a matching lang override replaces the scale- // wholesale — its own temperature_by_options first, then its 3-slot temperature —- // so an override without buckets intentionally ignores the base per-bucket entries.- const langCfg = lang ? this.langTemperatures[lang.split("-")[0].toLowerCase()] : undefined;- const scale = langCfg- ? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0)- : (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0);- const z = (logits[r] as number[]).slice(0, k).map((v) => v / scale);- const p = softmax(z);- const actRow = (act[r] as number[]) ?? [1, 0];- const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));- const ext = { act_probability: r4(actP[0]) };- // Same quantity on every question type (max(p)), so callers can gate across types on- // one number; `confidence` stays as-is for existing callers (entropy for choice/score).- const ansConf = r4(answerConfidence(p));- if (q.t === "choice") {- const keys = Object.keys(q.crit as Record<string, unknown>);- let best = 0;- for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i;- answers[qid] = {- type: "choice",- choice: keys[best],- probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),- confidence: r4(confidenceFromProbs(p)),- answer_confidence: ansConf,- action: ext,- };- } else if (q.t === "score") {- const exp = p.reduce((a, v, i) => a + i * v, 0);- answers[qid] = {- type: "score",- score: r4(exp),- // `renderCriterion`, not the raw `c`: a legend maps an index to the TEXT of a level, and- // the keys are already strings. A numeric scale written directly used to come back with- // the caller's own JSON types (`{"0": 1}`), so a client that reads a level as a string- // had to handle a number, a boolean and null as well -- and `{"0": null}` is not even- // parseable by a Jev client (#302). Same change as the Python `agent.py` and- // `onnx_agent.py` legends, so the two runtimes return the same value types.- legend: Object.fromEntries(- (q.crit as unknown[]).map((c, i) => [String(i), renderCriterion(c)]),- ),- probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),- confidence: r4(confidenceFromProbs(p)),- answer_confidence: ansConf,- action: ext,- };- } else {- const pt = p[1] ?? 0;- answers[qid] = {- type: "noul",- noul: r4(pt),- confidence: r4(Math.max(pt, 1 - pt)),- // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here- answer_confidence: ansConf,- action: ext,- };+ let row = 0;+ for (let s = 0; s < chunk.length; s++) {+ const answers: Record<string, SystemAnswer> = {};+ for (let qi = 0; qi < ids.length; qi++) {+ answers[ids[qi]] = this._decodeRow(+ logits[row] as number[],+ (act[row] as number[]) ?? [1, 0],+ internals[qi],+ built[s].items[qi].markers.length,+ opts.lang ?? null,+ );+ row++;+ }+ out.push({+ model: "laya-rl-agent",+ answers,+ usage: { input_tokens: built[s].nTokens, output_tokens: 0 },+ });}}- return { model: "laya-rl-agent", answers, usage: { input_tokens: nTokens, output_tokens: 0 } };+ return out;+ }++ /** Decode one head row into the answer for its question. */+ private _decodeRow(+ logitsRow: number[],+ actRow: number[],+ q: { t: "choice" | "score" | "noul"; ins: string; crit: unknown },+ k: number,+ lang: string | null = null,+ ): SystemAnswer {+ const qt = QTYPES[q.t];+ const bucket = tempBucket(qt, k);+ // Python parity (_decode_answers): a matching lang override replaces the scale+ // wholesale — its own temperature_by_options first, then its 3-slot temperature —+ // so an override without buckets intentionally ignores the base per-bucket entries.+ const langCfg = lang ? this.langTemperatures[lang.split("-")[0].toLowerCase()] : undefined;+ const scale = langCfg+ ? (langCfg.temperatureByOptions[bucket] ?? langCfg.temperature[qt] ?? 1.0)+ : (this.temperatureByOptions[bucket] ?? this.temperature[qt] ?? 1.0);+ const z = logitsRow.slice(0, k).map((v) => v / scale);+ const p = softmax(z);+ const actP = softmax(actRow.slice(0, Math.max(2, actRow.length)));+ const ext = { act_probability: r4(actP[0]) };+ // Same quantity on every question type (max(p)), so callers can gate across types on+ // one number; `confidence` stays as-is for existing callers (entropy for choice/score).+ const ansConf = r4(answerConfidence(p));+ if (q.t === "choice") {+ const keys = Object.keys(q.crit as Record<string, unknown>);+ let best = 0;+ for (let i = 1; i < p.length; i++) if (p[i] > p[best]) best = i;+ return {+ type: "choice",+ choice: keys[best],+ probabilities: Object.fromEntries(keys.map((kk, i) => [kk, r4(p[i] ?? 0)])),+ confidence: r4(confidenceFromProbs(p)),+ answer_confidence: ansConf,+ action: ext,+ };+ }+ if (q.t === "score") {+ const exp = p.reduce((a, v, i) => a + i * v, 0);+ return {+ type: "score",+ score: r4(exp),+ // `renderCriterion`, not the raw `c`: a legend maps an index to the TEXT of a level, and+ // the keys are already strings. A numeric scale written directly used to come back with+ // the caller's own JSON types (`{"0": 1}`), so a client that reads a level as a string+ // had to handle a number, a boolean and null as well -- and `{"0": null}` is not even+ // parseable by a Jev client (#302). Same change as the Python `agent.py` and+ // `onnx_agent.py` legends, so the two runtimes return the same value types.+ legend: Object.fromEntries(+ (q.crit as unknown[]).map((c, i) => [String(i), renderCriterion(c)]),+ ),+ probabilities: Object.fromEntries(p.map((v, i) => [String(i), r4(v)])),+ confidence: r4(confidenceFromProbs(p)),+ answer_confidence: ansConf,+ action: ext,+ };+ }+ const pt = p[1] ?? 0;+ return {+ type: "noul",+ noul: r4(pt),+ confidence: r4(Math.max(pt, 1 - pt)),+ // over two options max(p_true, 1 - p_true) is max(p): identical to confidence here+ answer_confidence: ansConf,+ action: ext,+ };}async predict(@@ -537,6 +611,25 @@ export class Agent extends HookRegistry {): Promise<SystemOneResult> {return this.systemOne(state, questions, opts);}+ /**+ * Evaluate the same questions over many states, packing each chunk of up to+ * `opts.batchSize` states into one shared forward pass (Python `Agent.predict_batch`+ * parity). This is the throughput path: `systemOne`/`predict` run one state per forward+ * pass, which leaves most of the batch dimension idle on GPU. Results align with+ * `states` by index and are identical in shape to `systemOne`'s output.+ *+ * `opts.maxLen` / `opts.headMaxLen` override the agent config for this call; a start+ * hook may also set `ctx.maxLen` / `ctx.headMaxLen` or call `ctx.skip(results)`.+ * Python's `sort_by_length` grouping is not ported.+ */+ async predictBatch(+ states: unknown[],+ questions: Record<string, QuestionDef>,+ opts: PredictOptions = {},+ ): Promise<SystemOneResult[]> {+ return this._predictHooked(states, questions, opts);+ }+/*** Answer `state` against a JSON schema (or explicit `opts.questions`) and return typeddiff --git a/laya-ts/src/index.ts b/laya-ts/src/index.tsindex 38e105a3..b311d389 100644--- a/laya-ts/src/index.ts+++ b/laya-ts/src/index.ts@@ -16,7 +16,7 @@ export type {export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle, loadWebBundle, PINNED_REVISIONS, resolveRevision } from "./providers.js";export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js";export { Router, normaliseName, DEFAULT_MODELS } from "./router.js";-export type { RoutedResult, RouteDecision, ModelName, ModelSpec } from "./router.js";+export type { RoutedResult, RouteDecision, ModelName, ModelSpec, BatchRequest } from "./router.js";export { shortlistChoice, predictShortlist, embedFnFromAgent, DEFAULT_SHORTLIST_K } from "./shortlist.js";export type { EmbedFn, ShortlistMeta } from "./shortlist.js";export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js";diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.tsindex 33ded388..fc968c0b 100644--- a/laya-ts/src/router.ts+++ b/laya-ts/src/router.ts@@ -10,6 +10,7 @@ import {markDefaultsRan,dispatchAsync,normaliseHooks,+ type Hook,type HookArg,type PredictHook,} from "./hooks.js";@@ -148,6 +149,30 @@ export interface RouteOptions {hooksRaise?: boolean;}+/** One item of a Router.predictBatch/routeBatch batch: state, questions, route overrides. */+export interface BatchRequest {+ state: unknown;+ questions: Record<string, QuestionDef>;+ model?: string | null;+ task?: string | null;+ lang?: string | null;+ langGuess?: LangGuess;+ lang_guess?: LangGuess;+}++function questionSchema(questions: Record<string, unknown>): string {+ // Python groups on json.dumps(sort_keys=False, default=str): insertion order is significant+ // at every nesting level because options are positional in the rendered sequence, so two+ // equal schemas with different key orders must not share a forward pass. JSON.stringify+ // preserves insertion order for string keys; the replacer stands in for default=str on+ // the few value types JSON cannot represent.+ return JSON.stringify(questions, (_key, value) =>+ typeof value === "function" || typeof value === "symbol" || typeof value === "bigint"+ ? String(value)+ : value,+ );+}+function toSpec(spec: string | ModelSpec | [string, string | null]): ModelSpec {if (typeof spec === "string") return { repo: spec, subfolder: null };if (Array.isArray(spec)) {@@ -579,4 +604,238 @@ export class Router extends HookRegistry {): Promise<RoutedResult> {return this.predict(state, questions, opts);}++ /**+ * Route many requests in one call (Python `Router.route_batch` parity). Every request is+ * validated before anything loads, so a malformed batch fails fast. Entries keep input+ * order; routing is deterministic, so repeated batches route identically.+ */+ routeBatch(requests: BatchRequest[]): RouteDecision[] {+ if (!Array.isArray(requests)) {+ throw new TypeError("requests must be an array of request objects");+ }+ const decisions: RouteDecision[] = [];+ for (let i = 0; i < requests.length; i++) {+ const request = requests[i];+ if (typeof request !== "object" || request === null || Array.isArray(request)) {+ const got = Array.isArray(request) ? "list" : request === null ? "null" : typeof request;+ throw new TypeError(`request ${i} must be an object, got ${got}`);+ }+ if (!("state" in request)) {+ throw new Error(`request ${i} is missing required key 'state'`);+ }+ if (!("questions" in request)) {+ throw new Error(`request ${i} is missing required key 'questions'`);+ }+ const questions = request.questions;+ if (typeof questions !== "object" || questions === null || Array.isArray(questions)) {+ const got =+ Array.isArray(questions) ? "list" : questions === null ? "null" : typeof questions;+ throw new TypeError(`request ${i} 'questions' must be an object, got ${got}`);+ }+ decisions.push(+ this.route(request.state, questions, {+ model: request.model ?? null,+ task: request.task ?? null,+ lang: request.lang ?? null,+ langGuess: request.langGuess ?? request.lang_guess ?? null,+ }),+ );+ }+ return decisions;+ }++ /**+ * Route and run many requests in one call (Python `Router.predict_batch` parity).+ * Requests are routed first, then grouped by checkpoint so each loaded Agent scores its+ * requests in as few forward passes as possible; results are restored to input order.+ * Requests routed to the same checkpoint still split into separate `predictBatch` calls+ * when their question schemas differ (order-sensitively) or when per-request start hooks+ * set different maxLen/headMaxLen overrides.+ *+ * Router-level predict hooks run per request: each request gets its own PredictContext+ * carrying `decision`; `onPredictStart` may rewrite a request or `ctx.skip()` it, and+ * `onPredictEnd` runs once per started request even when the batch fails.+ */+ async predictBatch(+ requests: BatchRequest[],+ batchSize: number | null = null,+ ): Promise<RoutedResult[]> {+ const decisions = this.routeBatch(requests);+ if (decisions.length === 0) return [];++ const groups = new Map<string, number[]>();+ for (let i = 0; i < decisions.length; i++) {+ const model = decisions[i].model;+ const bucket = groups.get(model);+ if (bucket) bucket.push(i);+ else groups.set(model, [i]);+ }++ const results: (RoutedResult | null)[] = new Array(requests.length).fill(null);+ const active = composeHooks(this.hooks);+ const raiseErrors = this.hooksRaise;++ for (const [modelName, indices] of groups) {+ const agent = (await this.load(modelName)) as {+ predictBatch?(+ states: unknown[],+ questions: Record<string, QuestionDef>,+ opts?: { batchSize?: number | null; maxLen?: number | null; headMaxLen?: number | null },+ ): Promise<SystemOneResult[]>;+ systemOne(+ state: unknown,+ questions: Record<string, QuestionDef>,+ ): Promise<SystemOneResult>;+ };+ const started: PredictContext[] = [];+ try {+ for (const i of indices) {+ const ctx = new PredictContext({+ states: [requests[i].state],+ questions: requests[i].questions as Record<string, unknown>,+ decision: { ...decisions[i] } as unknown as Record<string, unknown>,+ model: modelName,+ agent,+ router: this,+ });+ started.push(ctx);+ dispatch(active, "onPredictStart", ctx, { raiseErrors });+ }++ // Order-sensitive at every nesting level (#166): options are positional, so two+ // equal schemas with different key orders must not share a group.+ const questionGroups: {+ questions: Record<string, unknown>;+ schema: string;+ maxLen: number | null;+ headMaxLen: number | null;+ items: [number, PredictContext][];+ }[] = [];+ for (let s = 0; s < indices.length; s++) {+ const i = indices[s];+ const ctx = started[s];+ if (ctx.results !== null) {+ // A cache hit short-circuits inference, but predict still promises a `routing`+ // key. Add it without overwriting a routing the cached payload already has.+ for (const result of ctx.results) {+ if (result && typeof result === "object" && !("routing" in result)) {+ (result as unknown as RoutedResult).routing = { ...decisions[i] };+ }+ }+ continue;+ }+ const schema = questionSchema(ctx.questions);+ const found = questionGroups.find(+ (g) => g.schema === schema && g.maxLen === ctx.maxLen && g.headMaxLen === ctx.headMaxLen,+ );+ if (found) found.items.push([i, ctx]);+ else {+ questionGroups.push({+ questions: ctx.questions,+ schema,+ maxLen: ctx.maxLen,+ headMaxLen: ctx.headMaxLen,+ items: [[i, ctx]],+ });+ }+ }++ for (const group of questionGroups) {+ const agentOpts: {+ batchSize: number | null;+ maxLen?: number;+ headMaxLen?: number;+ } = { batchSize };+ // Only pass overrides when set, so an Agent-like object that does not accept+ // them still works.+ if (group.maxLen !== null) agentOpts.maxLen = group.maxLen;+ if (group.headMaxLen !== null) agentOpts.headMaxLen = group.headMaxLen;+ let batchResults: SystemOneResult[];+ if (typeof agent.predictBatch === "function") {+ batchResults = await agent.predictBatch(+ group.items.map(([, ctx]) => ctx.states[0]),+ group.questions as Record<string, QuestionDef>,+ agentOpts,+ );+ } else {+ // Agent-like objects that only implement systemOne (light wrappers, tests)+ // still work; their states simply share no forward pass.+ batchResults = [];+ for (const [, ctx] of group.items) {+ batchResults.push(+ await agent.systemOne(ctx.states[0], group.questions as Record<string, QuestionDef>),+ );+ }+ }+ if (batchResults.length !== group.items.length) {+ throw new Error(+ `internal error: Agent.predictBatch returned ${batchResults.length} results for ${group.items.length} states`,+ );+ }+ for (let s = 0; s < group.items.length; s++) {+ const [i, ctx] = group.items[s];+ const result = batchResults[s] as RoutedResult;+ result.routing = { ...decisions[i] };+ ctx.results = [result as unknown as Record<string, unknown>];+ }+ }+ } catch (err) {+ for (const ctx of started) {+ if (ctx.results === null) {+ ctx.error = err;+ try {+ dispatch(active, "onError", ctx, { raiseErrors });+ } catch {+ // Chained onto the batch failure, which is the one that propagates.+ }+ }+ }+ try {+ this._endContexts(active, started, raiseErrors);+ } catch {+ // A failing end hook is chained onto the batch failure, not raised instead.+ }+ throw err;+ }+ this._endContexts(active, started, raiseErrors);+ for (let s = 0; s < indices.length; s++) {+ results[indices[s]] = started[s].results![0] as unknown as RoutedResult;+ }+ }++ if (results.some((r) => r === null)) {+ throw new Error("internal error: batch execution did not produce every result");+ }+ return results as RoutedResult[];+ }++ /** Alias of predictBatch, mirroring Python's `Router.predict_many`. */+ async predictMany(+ requests: BatchRequest[],+ batchSize: number | null = null,+ ): Promise<RoutedResult[]> {+ return this.predictBatch(requests, batchSize);+ }++ /**+ * Run onPredictEnd for every started per-request context and raise the first hook+ * failure, if any. Elapsed and usage are stamped on every context before any end hook+ * runs, so one request's end hooks never inflate another request's elapsed time.+ */+ private _endContexts(active: Hook[], contexts: PredictContext[], raiseErrors: boolean): void {+ for (const ctx of contexts) {+ ctx.markElapsed();+ if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);+ }+ let firstError: unknown = null;+ for (const ctx of contexts) {+ try {+ dispatch(active, "onPredictEnd", ctx, { raiseErrors });+ } catch (hookErr) {+ if (ctx.error === null && firstError === null) firstError = hookErr;+ }+ }+ if (firstError !== null) throw firstError;+ }}