Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(ts): port the Python hooks lifecycle to laya-ts (#308)

Aashish Mahato2e9bcc540fd2
diff --git a/laya-ts/README.md b/laya-ts/README.md
index b088b794..e41042f8 100644
--- a/laya-ts/README.md
+++ b/laya-ts/README.md
@@ -1,6 +1,6 @@
# laya-ts
-TypeScript inference for Laya (`Agent.predict`, `Router`, `lang`, `email`, `presets`, `shortlist`) on Node and the browser via split ONNX (`encoder.onnx` + `head.onnx`). ESM-only (`"type": "module"`); no CJS build — import from ESM or bundle.
+TypeScript inference for Laya (`Agent.predict`, `Router`, `lang`, `email`, `presets`, `shortlist`, `hooks`) on Node and the browser via split ONNX (`encoder.onnx` + `head.onnx`). ESM-only (`"type": "module"`); no CJS build — import from ESM or bundle.
## Export weights (once per checkpoint)
@@ -39,6 +39,25 @@ const out = await agent.predict("charged twice", {
`onnxruntime-node` / `onnxruntime-web` are optional peer deps, imported lazily behind the provider you use.
+## Hooks (observe or shape every decision)
+
+Port of the Python `laya.hooks` lifecycle. A hook is a `(ctx) => void` for `onPredictStart` /
+`onPredictEnd`, or an object implementing any subset of `onPredictStart`, `onPredictEnd`,
+`onRoute`, `onLoad`, `onEvict`, `onError`:
+
+```ts
+const tracer = {
+ onPredictStart(ctx) { console.time(ctx.runId); },
+ onPredictEnd(ctx) { console.timeEnd(ctx.runId); console.log(ctx.model, ctx.usage, ctx.elapsedMs); },
+};
+const router = new Router({ hooks: [tracer], hooksRaise: false }); // telemetry must not fail a request
+await router.withHooks([auditHook], () => router.predict(state, questions)); // scoped install
+
+// a start hook may rewrite ctx.states / ctx.questions, or serve a cached result:
+const cache = { onPredictStart(ctx) { const hit = lookup(ctx.states[0]); if (hit) ctx.skip([hit]); } };
+// an onRoute hook may replace ctx.decision (e.g. pin a checkpoint)
+```
+
## Shortlist (many labels)
```ts
diff --git a/laya-ts/src/agent.ts b/laya-ts/src/agent.ts
index 4e732b38..49a65cd8 100644
--- a/laya-ts/src/agent.ts
+++ b/laya-ts/src/agent.ts
@@ -13,6 +13,15 @@ import {
} from "./common.js";
import type { Batch, SessionProvider } from "./providers.js";
import { encodeWithData, parseTokenizerJson, type TokenizerLike } from "./tokenizer.js";
+import {
+ HookRegistry,
+ PredictContext,
+ aggregateUsage,
+ dispatch,
+ normaliseHooks,
+ type HookArg,
+ type PredictHook,
+} from "./hooks.js";
export const QTYPES: Record<string, number> = { choice: 0, score: 1, noul: 2 };
@@ -80,6 +89,18 @@ export interface AgentOptions {
head_max_len?: number;
temperature?: unknown;
temperature_by_options?: Record<string, unknown>;
+ hooks?: HookArg;
+ onPredictStart?: PredictHook;
+ onPredictEnd?: PredictHook;
+ hooksRaise?: boolean;
+}
+
+/** Per-call hook options shared by Agent.systemOne/predict and Router.predict. */
+export interface PredictOptions {
+ hooks?: HookArg;
+ onPredictStart?: PredictHook;
+ onPredictEnd?: PredictHook;
+ hooksRaise?: boolean;
}
function qidStr(qid: string): string {
@@ -192,7 +213,8 @@ function tokenizerFromConfig(cfg: AgentCfg): TokenizerLike | null {
const r4 = (v: number): number => Math.round(v * 1e4) / 1e4;
-export class Agent {
+export class Agent extends HookRegistry {
+ hooksRaise: boolean;
cfg: AgentCfg;
provider: SessionProvider;
tok: TokenizerLike;
@@ -204,8 +226,12 @@ export class Agent {
temperatureByOptions: Record<string, number>;
constructor(opts: AgentOptions) {
+ super();
if (!opts || !opts.provider) throw new Error("Agent needs a provider");
this.provider = opts.provider;
+ // Hooks are opt-in; an unset hook list is a no-op. See hooks.ts.
+ this.hooks = normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd);
+ this.hooksRaise = opts.hooksRaise ?? true;
const cfg = { ...(opts.cfg ?? {}) } as AgentCfg;
if (opts.max_len !== undefined) cfg.max_len = opts.max_len;
if (opts.head_max_len !== undefined) cfg.head_max_len = opts.head_max_len;
@@ -244,7 +270,72 @@ export class Agent {
}
}
- async systemOne(state: unknown, questions: Record<string, QuestionDef>): Promise<SystemOneResult> {
+ /**
+ * Evaluate typed questions across one state in a single forward pass.
+ *
+ * `hooks` / `onPredictStart` / `onPredictEnd` observe or shape the prediction, appended
+ * after any hooks installed on the Agent; a start hook may rewrite the state/questions or
+ * call `ctx.skip(...)` to short-circuit inference, an end hook may rewrite the results.
+ * See hooks.ts. `hooksRaise` overrides the Agent's setting for this call.
+ */
+ async systemOne(
+ state: unknown,
+ questions: Record<string, QuestionDef>,
+ opts: PredictOptions = {},
+ ): Promise<SystemOneResult> {
+ return (await this._predictHooked([state], questions, opts))[0];
+ }
+
+ private async _predictHooked(
+ states: unknown[],
+ questions: Record<string, QuestionDef>,
+ opts: PredictOptions,
+ ): Promise<SystemOneResult[]> {
+ const active = [
+ ...this.hooks,
+ ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd),
+ ];
+ const raiseErrors = opts.hooksRaise ?? this.hooksRaise;
+ const ctx = new PredictContext({
+ states,
+ questions: questions as Record<string, unknown>,
+ agent: this,
+ });
+ try {
+ dispatch(active, "onPredictStart", ctx, { raiseErrors });
+ if (ctx.results === null) {
+ const out: SystemOneResult[] = [];
+ for (const st of ctx.states) {
+ out.push(await this._systemOneCore(st, ctx.questions as Record<string, QuestionDef>));
+ }
+ ctx.results = out as unknown as Record<string, unknown>[];
+ ctx.model ??= out[0]?.model ?? null;
+ }
+ } catch (err) {
+ ctx.error = err;
+ try {
+ dispatch(active, "onError", ctx, { raiseErrors });
+ } catch {
+ // A failing onError hook must not hide the failure that triggered it.
+ }
+ throw err;
+ } finally {
+ ctx.markElapsed();
+ if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);
+ try {
+ dispatch(active, "onPredictEnd", ctx, { raiseErrors });
+ } catch (hookErr) {
+ // End hooks run on the failure path too; do not let one mask the real error.
+ if (ctx.error === null) throw hookErr;
+ }
+ }
+ return ctx.results as unknown as SystemOneResult[];
+ }
+
+ private async _systemOneCore(
+ state: unknown,
+ questions: Record<string, QuestionDef>,
+ ): Promise<SystemOneResult> {
const ids = Object.keys(questions ?? {});
if (ids.length === 0) {
return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } };
@@ -319,8 +410,12 @@ export class Agent {
return { model: "laya-rl-agent", answers, usage: { input_tokens: nTokens, output_tokens: 0 } };
}
- async predict(state: unknown, questions: Record<string, QuestionDef>): Promise<SystemOneResult> {
- return this.systemOne(state, questions);
+ async predict(
+ state: unknown,
+ questions: Record<string, QuestionDef>,
+ opts: PredictOptions = {},
+ ): Promise<SystemOneResult> {
+ return this.systemOne(state, questions, opts);
}
static async load(
diff --git a/laya-ts/src/hooks.ts b/laya-ts/src/hooks.ts
new file mode 100644
index 00000000..59bbffdf
--- /dev/null
+++ b/laya-ts/src/hooks.ts
@@ -0,0 +1,260 @@
+/**
+ * Opt-in prediction hooks: observe or shape every decision without forking.
+ *
+ * A hook is either a plain `(ctx) => void` or an object implementing any subset of the
+ * lifecycle methods on `Hook`. Hooks are configured on `Agent` / `Router` and can be
+ * overridden per call. Port of `laya/hooks.py`.
+ */
+
+/** Mutable state passed to every hook for one call. */
+export class PredictContext {
+ /** `states` / `questions` may be rewritten by `onPredictStart`. */
+ states: unknown[];
+ questions: Record<string, unknown>;
+ /** Shared by every hook of one call. */
+ runId: string;
+ /** Set by `skip()` or after inference; `onPredictEnd` may rewrite it. */
+ results: Record<string, unknown>[] | null = null;
+ /** Router: the RouteDecision. */
+ decision: Record<string, unknown> | null = null;
+ /** Resolved checkpoint name. */
+ model: string | null = null;
+ agent: unknown = null;
+ router: unknown = null;
+ /** Per-call token-budget overrides; null = agent config. */
+ maxLen: number | null = null;
+ headMaxLen: number | null = null;
+ /** Aggregated input/output tokens across results. */
+ usage: Record<string, number> | null = null;
+ startedAt: number;
+ elapsedMs: number | null = null;
+ error: unknown = null;
+
+ constructor(init: { states: unknown[]; questions: Record<string, unknown> } & Partial<PredictContext>) {
+ this.states = init.states;
+ this.questions = init.questions;
+ this.runId = newRunId();
+ this.startedAt = now();
+ for (const [k, v] of Object.entries(init)) {
+ if (k !== "states" && k !== "questions" && v !== undefined) {
+ (this as unknown as Record<string, unknown>)[k] = v;
+ }
+ }
+ }
+
+ /** Set cached results from a start hook; inference is skipped, end hooks still run. */
+ skip(results: Record<string, unknown>[]): void {
+ this.results = results;
+ }
+
+ /** Set elapsedMs from startedAt. Called by the dispatching call, not by hooks. */
+ markElapsed(): void {
+ this.elapsedMs = now() - this.startedAt;
+ }
+}
+
+function newRunId(): string {
+ const c = (globalThis as { crypto?: { randomUUID?: () => string } }).crypto;
+ if (c && typeof c.randomUUID === "function") return c.randomUUID().replaceAll("-", "");
+ let hex = "";
+ for (let i = 0; i < 32; i++) hex += Math.floor(Math.random() * 16).toString(16);
+ return hex;
+}
+
+function now(): number {
+ const p = (globalThis as { performance?: { now?: () => number } }).performance;
+ return p && typeof p.now === "function" ? p.now() : Date.now();
+}
+
+/** Optional lifecycle methods. Implement any subset; missing methods are skipped. */
+export interface Hook {
+ onPredictStart?(ctx: PredictContext): void;
+ onPredictEnd?(ctx: PredictContext): void;
+ onRoute?(ctx: PredictContext): void;
+ onLoad?(ctx: PredictContext): void;
+ onEvict?(ctx: PredictContext): void;
+ onError?(ctx: PredictContext): void;
+}
+
+export type PredictHook = (ctx: PredictContext) => void;
+export type HookArg = Hook | Hook[] | null | undefined;
+export type PredictHookArg = PredictHook | PredictHook[] | null | undefined;
+
+export const HOOK_EVENTS = [
+ "onPredictStart",
+ "onPredictEnd",
+ "onRoute",
+ "onLoad",
+ "onEvict",
+ "onError",
+] as const;
+export type HookEvent = (typeof HOOK_EVENTS)[number];
+
+function isHookLike(item: unknown): item is Hook {
+ return (
+ typeof item === "object" &&
+ item !== null &&
+ HOOK_EVENTS.some((e) => typeof (item as Record<string, unknown>)[e] === "function")
+ );
+}
+
+function typeName(v: unknown): string {
+ if (v === null) return "null";
+ if (Array.isArray(v)) return "Array";
+ if (typeof v === "object") return (v as object).constructor?.name ?? "object";
+ return typeof v;
+}
+
+/**
+ * Normalise the hook arguments shared by Agent, Router and every predict call: an optional
+ * hook (object or list) plus optional plain start/end callables. A plain callable is not a
+ * hook on its own -- which lifecycle event would it bind to? Passing one as `hooks` is a
+ * TypeError; pass it as `onPredictStart` / `onPredictEnd` instead. The same callable as both
+ * start and end is a TypeError too: the registry dedupes by identity, so one of the two would
+ * silently never fire.
+ */
+export function normaliseHooks(
+ hooks?: unknown,
+ onPredictStart?: PredictHookArg,
+ onPredictEnd?: PredictHookArg,
+): Hook[] {
+ const out: Hook[] = [];
+ const append = (item: unknown): void => {
+ if (item === null || item === undefined) return;
+ if (typeof item === "function") {
+ // A JS function can carry properties; one that also defines hook methods is ambiguous.
+ if (HOOK_EVENTS.some((e) => typeof (item as unknown as Record<string, unknown>)[e] === "function")) {
+ throw new TypeError(
+ "a hook must be a plain callable or implement hook methods, not both",
+ );
+ }
+ throw new TypeError(
+ "a plain callable is ambiguous; pass it as onPredictStart or onPredictEnd",
+ );
+ }
+ if (Array.isArray(item)) {
+ for (const h of item) append(h);
+ return;
+ }
+ if (typeof item !== "object") {
+ throw new TypeError(`hooks must be Hook objects, got ${typeName(item)}`);
+ }
+ if (!isHookLike(item)) {
+ throw new TypeError(
+ `a hook must implement at least one hook method (${HOOK_EVENTS.join(", ")})`,
+ );
+ }
+ out.push(item as Hook);
+ };
+ append(hooks);
+ if (onPredictStart !== null && onPredictStart !== undefined) {
+ if (typeof onPredictStart !== "function") {
+ throw new TypeError(`onPredictStart must be callable, got ${typeName(onPredictStart)}`);
+ }
+ out.push({ onPredictStart });
+ }
+ if (onPredictEnd !== null && onPredictEnd !== undefined) {
+ if (typeof onPredictEnd !== "function") {
+ throw new TypeError(`onPredictEnd must be callable, got ${typeName(onPredictEnd)}`);
+ }
+ out.push({ onPredictEnd });
+ }
+ if (
+ onPredictStart !== null &&
+ onPredictStart !== undefined &&
+ onPredictStart === (onPredictEnd as unknown)
+ ) {
+ throw new TypeError(
+ "onPredictStart and onPredictEnd must be different callables; one callable cannot serve both events",
+ );
+ }
+ return out;
+}
+
+/**
+ * Base class giving a runtime-mutable hook list.
+ *
+ * `Agent` and `Router` extend it so hooks can be added, removed or scoped after construction.
+ * Mutation replaces the array, so a call that already snapshotted its active hooks is never
+ * disturbed by an add or remove that lands mid-flight.
+ */
+export class HookRegistry {
+ hooks: Hook[] = [];
+
+ /** Install one hook or a list of them. Returns this for chaining. */
+ addHook(hook: HookArg): this {
+ const added = normaliseHooks(hook);
+ if (added.length > 0) this.hooks = [...this.hooks, ...added];
+ return this;
+ }
+
+ /** Remove a hook by identity. Returns true if it was installed. */
+ removeHook(hook: unknown): boolean {
+ const before = this.hooks.length;
+ this.hooks = this.hooks.filter((installed) => installed !== hook);
+ return this.hooks.length !== before;
+ }
+
+ /**
+ * Run `fn` with hooks installed, then remove them -- works for sync and async `fn`:
+ *
+ * await router.withHooks([tracer], () => router.predict(state, questions));
+ */
+ withHooks<T>(hookArgs: HookArg[], fn: () => T): T {
+ const added = normaliseHooks(hookArgs);
+ this.hooks = [...this.hooks, ...added];
+ const release = (): void => {
+ this.hooks = this.hooks.filter((installed) => !added.includes(installed));
+ };
+ let out: T;
+ try {
+ out = fn();
+ } catch (err) {
+ release();
+ throw err;
+ }
+ if (out instanceof Promise) {
+ return out.finally(release) as T;
+ }
+ release();
+ return out;
+ }
+}
+
+/** Sum the per-state usage blocks so a hook sees one total for the call. */
+export function aggregateUsage(results: Record<string, unknown>[]): Record<string, number> {
+ const total = (key: string): number =>
+ results.reduce((acc, r) => {
+ const usage = r?.["usage"] as Record<string, unknown> | null | undefined;
+ const v = usage?.[key];
+ return acc + (typeof v === "number" && Number.isFinite(v) ? Math.trunc(v) : 0);
+ }, 0);
+ return { input_tokens: total("input_tokens"), output_tokens: total("output_tokens") };
+}
+
+/**
+ * Call `event` on every hook that implements it.
+ *
+ * `raiseErrors=false` warns and continues, for hooks (telemetry) that must not fail a
+ * request. Python also takes a `lock` to serialise dispatch for hooks that are not safe to
+ * run concurrently; JS hooks run synchronously on one thread, so there is nothing to lock.
+ */
+export function dispatch(
+ hooks: Hook[],
+ event: HookEvent,
+ ctx: PredictContext,
+ opts: { raiseErrors?: boolean } = {},
+): void {
+ const raiseErrors = opts.raiseErrors ?? true;
+ for (const hook of hooks) {
+ const method = hook?.[event];
+ if (typeof method !== "function") continue;
+ try {
+ method.call(hook, ctx);
+ } catch (err) {
+ if (raiseErrors) throw err;
+ const name = (hook as object)?.constructor?.name ?? "hook";
+ console.warn(`laya: hook ${name}.${event} failed: ${String(err)}`);
+ }
+ }
+}
diff --git a/laya-ts/src/index.ts b/laya-ts/src/index.ts
index 7496e360..33f16abc 100644
--- a/laya-ts/src/index.ts
+++ b/laya-ts/src/index.ts
@@ -11,6 +11,7 @@ export type {
ActionInfo,
AgentCfg,
AgentOptions,
+ PredictOptions,
} from "./agent.js";
export { createNodeProvider, createWebProvider, feed, feedHead, loadNodeBundle, loadWebBundle } from "./providers.js";
export type { Batch, SessionProvider, ProviderOptions, NodeBundle, WebBundle } from "./providers.js";
@@ -21,6 +22,8 @@ export type { EmbedFn, ShortlistMeta } from "./shortlist.js";
export { analyse, isEnglish, guessLatinLanguage, detectScript } from "./lang.js";
export type { AnalyseResult, LatinProfile } from "./lang.js";
export { cleanEmailBody, emailState } from "./email.js";
+export { PredictContext, HookRegistry, normaliseHooks, dispatch, aggregateUsage, HOOK_EVENTS } from "./hooks.js";
+export type { Hook, HookEvent, HookArg, PredictHook, PredictHookArg } from "./hooks.js";
export { triageQuestions, emailQuestions, guardQuestions, moderationQuestions, routerQuestions } from "./presets.js";
export {
renderOptions,
diff --git a/laya-ts/src/router.ts b/laya-ts/src/router.ts
index 009cbeec..8e21c218 100644
--- a/laya-ts/src/router.ts
+++ b/laya-ts/src/router.ts
@@ -1,5 +1,14 @@
import { analyse, type AnalyseResult } from "./lang.js";
-import type { QuestionDef, SystemOneResult } from "./agent.js";
+import type { PredictOptions, QuestionDef, SystemOneResult } from "./agent.js";
+import {
+ HookRegistry,
+ PredictContext,
+ aggregateUsage,
+ dispatch,
+ normaliseHooks,
+ type HookArg,
+ type PredictHook,
+} from "./hooks.js";
export const BUNDLE_REPO = "convaiinnovations/laya";
@@ -109,6 +118,10 @@ export interface RouterOptions {
langGuess?: LangGuess;
lang_guess?: LangGuess;
loader?: AgentLoader;
+ hooks?: HookArg;
+ onPredictStart?: PredictHook;
+ onPredictEnd?: PredictHook;
+ hooksRaise?: boolean;
}
export interface RouteOptions {
@@ -117,6 +130,8 @@ export interface RouteOptions {
lang?: string | null;
langGuess?: LangGuess;
lang_guess?: LangGuess;
+ hooks?: HookArg;
+ hooksRaise?: boolean;
}
function toSpec(spec: string | ModelSpec | [string, string | null]): ModelSpec {
@@ -132,7 +147,8 @@ function repoStr(spec: ModelSpec): string {
return spec.subfolder ? `${spec.repo}/${spec.subfolder}` : spec.repo;
}
-export class Router {
+export class Router extends HookRegistry {
+ hooksRaise: boolean;
models: Record<string, ModelSpec>;
device: string | null;
token: string | null | undefined;
@@ -145,6 +161,11 @@ export class Router {
_order: string[] = []; // least-recently-used first
constructor(opts: RouterOptions = {}) {
+ super();
+ // Hooks are opt-in; an unset hook list is a no-op. Router-level onPredictStart /
+ // onPredictEnd hooks wrap the whole route+infer call and see ctx.decision; see hooks.ts.
+ this.hooks = normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd);
+ this.hooksRaise = opts.hooksRaise ?? true;
const base: Record<string, string | ModelSpec> = opts.standaloneRepos ?? opts.standalone_repos
? { ...STANDALONE_MODELS }
: Object.fromEntries(Object.entries(DEFAULT_MODELS).map(([k, v]) => [k, { ...v }]));
@@ -190,7 +211,22 @@ export class Router {
}
this._agents.set(key, agent);
this._order.push(key);
- this._evict();
+ const evicted = this._evict();
+ // Lifecycle hooks fire after the maps settle, so a hook can safely call the Router.
+ for (const victim of evicted) {
+ dispatch(
+ this.hooks,
+ "onEvict",
+ new PredictContext({ states: [], questions: {}, model: victim, router: this }),
+ { raiseErrors: this.hooksRaise },
+ );
+ }
+ dispatch(
+ this.hooks,
+ "onLoad",
+ new PredictContext({ states: [], questions: {}, model: key, agent, router: this }),
+ { raiseErrors: this.hooksRaise },
+ );
return agent;
}
@@ -200,17 +236,23 @@ export class Router {
this._order.push(key);
}
- _evict(): void {
+ /** Drop least-recently-used agents until `maxLoaded` holds. Returns evicted names. */
+ _evict(): string[] {
+ const evicted: string[] = [];
while (this._order.length > this.maxLoaded) {
const victim = this._order.shift()!;
- this._agents.delete(victim);
+ if (this._agents.delete(victim)) evicted.push(victim);
}
// Keep the two views consistent.
if (this._order.length < this._agents.size) {
for (const k of [...this._agents.keys()]) {
- if (!this._order.includes(k)) this._agents.delete(k);
+ if (!this._order.includes(k)) {
+ this._agents.delete(k);
+ evicted.push(k);
+ }
}
}
+ return evicted;
}
attach(name: string, agent: unknown): unknown {
@@ -252,10 +294,36 @@ export class Router {
return englishFromCode(value);
}
+ /**
+ * Decide which checkpoint to use, then let `onRoute` hooks observe or replace the decision.
+ *
+ * `ctx.decision` is the RouteDecision; a hook may replace it (for example to pin a
+ * checkpoint) and the replacement is what gets returned and used. `opts.hooks` are
+ * per-call hooks, appended after any installed on the Router.
+ */
route(
state: unknown,
questions: Record<string, unknown> | null = null,
opts: RouteOptions = {},
+ ): RouteDecision {
+ const decision = this._route(state, questions, opts);
+ const raiseErrors = opts.hooksRaise ?? this.hooksRaise;
+ const active = [...this.hooks, ...normaliseHooks(opts.hooks)];
+ const ctx = new PredictContext({
+ states: [state],
+ questions: (questions ?? {}) as Record<string, unknown>,
+ decision: decision as unknown as Record<string, unknown>,
+ router: this,
+ });
+ dispatch(active, "onRoute", ctx, { raiseErrors });
+ return ctx.decision as unknown as RouteDecision;
+ }
+
+ /** Decide which checkpoint to use, without loading, running, or hooking anything. */
+ _route(
+ state: unknown,
+ questions: Record<string, unknown> | null = null,
+ opts: RouteOptions = {},
): RouteDecision {
const { model = null, task = null, lang = null } = opts;
const langGuessOpt = opts.langGuess ?? opts.lang_guess ?? null;
@@ -352,24 +420,80 @@ export class Router {
return { model: key, repo: repoStr(this.models[key]), reason, detection: det, workflow };
}
+ /**
+ * Route, then answer every question in one forward pass on the chosen checkpoint.
+ *
+ * The result is the usual systemOne payload plus a `routing` key recording the decision.
+ * Router-level `onPredictStart` / `onPredictEnd` hooks wrap the whole route+infer call and
+ * see `ctx.decision`; see hooks.ts.
+ */
async predict(
state: unknown,
questions: Record<string, QuestionDef>,
- opts: RouteOptions = {},
+ opts: RouteOptions & PredictOptions = {},
): Promise<RoutedResult> {
+ const active = [
+ ...this.hooks,
+ ...normaliseHooks(opts.hooks, opts.onPredictStart, opts.onPredictEnd),
+ ];
+ const raiseErrors = opts.hooksRaise ?? this.hooksRaise;
+
+ // Per-call hooks apply to the whole call, including onRoute inside route().
const decision = this.route(state, questions, opts);
const agent = (await this.load(decision.model)) as {
systemOne(s: unknown, q: Record<string, QuestionDef>): Promise<SystemOneResult>;
};
- const result = (await agent.systemOne(state, questions)) as RoutedResult;
- result["routing"] = { ...decision };
- return result;
+ const ctx = new PredictContext({
+ states: [state],
+ questions: questions as Record<string, unknown>,
+ decision: { ...decision } as unknown as Record<string, unknown>,
+ model: decision.model,
+ agent,
+ router: this,
+ });
+ try {
+ dispatch(active, "onPredictStart", ctx, { raiseErrors });
+ if (ctx.results === null) {
+ const result = (await agent.systemOne(
+ ctx.states[0],
+ ctx.questions as Record<string, QuestionDef>,
+ )) as RoutedResult;
+ result["routing"] = { ...decision };
+ ctx.results = [result as unknown as Record<string, unknown>];
+ } else {
+ // A cache hit short-circuits inference, but predict still promises a `routing` key.
+ // Add it without overwriting a routing the cached payload already has.
+ for (const result of ctx.results) {
+ if (result && typeof result === "object" && !("routing" in result)) {
+ (result as unknown as RoutedResult).routing = { ...decision };
+ }
+ }
+ }
+ } catch (err) {
+ ctx.error = err;
+ try {
+ dispatch(active, "onError", ctx, { raiseErrors });
+ } catch {
+ // A failing onError hook must not hide the failure that triggered it.
+ }
+ throw err;
+ } finally {
+ ctx.markElapsed();
+ if (ctx.results !== null) ctx.usage = aggregateUsage(ctx.results);
+ try {
+ dispatch(active, "onPredictEnd", ctx, { raiseErrors });
+ } catch (hookErr) {
+ // End hooks run on the failure path too; do not let one mask the real error.
+ if (ctx.error === null) throw hookErr;
+ }
+ }
+ return (ctx.results as unknown as RoutedResult[])[0];
}
async systemOne(
state: unknown,
questions: Record<string, QuestionDef>,
- opts: RouteOptions = {},
+ opts: RouteOptions & PredictOptions = {},
): Promise<RoutedResult> {
return this.predict(state, questions, opts);
}
diff --git a/laya-ts/tests/hooks.test.ts b/laya-ts/tests/hooks.test.ts
new file mode 100644
index 00000000..10d39690
--- /dev/null
+++ b/laya-ts/tests/hooks.test.ts
@@ -0,0 +1,300 @@
+import { describe, expect, it, vi } from "vitest";
+import { Agent, defaultTokenizer } from "../src/agent.js";
+import { Router } from "../src/router.js";
+import { HookRegistry, normaliseHooks, type Hook, type PredictContext } from "../src/hooks.js";
+
+const QUESTIONS = { q: { type: "noul", instructions: "?" } } as never;
+
+/** A real Agent with the model replaced by a recording fake provider. */
+function makeAgent(seen: string[] = [], opts: Record<string, unknown> = {}) {
+ const provider = {
+ async runEncoder(_b: unknown) {
+ return { lastHidden: [[1, 0], [0, 1]] };
+ },
+ async runHead(_h: unknown) {
+ return { logits: [[2, 0]], act: [[3, 0]] };
+ },
+ };
+ const tok = {
+ ...defaultTokenizer(),
+ encode(text: string): number[] {
+ seen.push(text);
+ return text.split(/\s+/).filter(Boolean).map(() => 7);
+ },
+ };
+ return new Agent({ provider, tok, ...opts } as never);
+}
+
+function makeRouter(extra: Record<string, unknown> = {}) {
+ const calls: unknown[] = [];
+ const loader = (_name: string) => ({
+ async systemOne(state: unknown, _q: unknown) {
+ calls.push(state);
+ return { model: "laya-rl-agent", answers: {}, usage: { input_tokens: 5, output_tokens: 0 } };
+ },
+ });
+ const router = new Router({ loader, ...extra } as never);
+ return { router, calls };
+}
+
+describe("agent hooks", () => {
+ it("start and end hooks fire once each with a shared context", async () => {
+ const events: string[] = [];
+ let startCtx: PredictContext | null = null;
+ const hook: Hook = {
+ onPredictStart(ctx) {
+ events.push("start");
+ startCtx = ctx;
+ },
+ onPredictEnd(ctx) {
+ events.push("end");
+ expect(ctx.runId).toBe((startCtx as PredictContext | null)?.runId);
+ expect(ctx.results).not.toBeNull();
+ expect(ctx.elapsedMs).not.toBeNull();
+ expect(ctx.usage?.input_tokens).toBeGreaterThan(0);
+ expect(ctx.agent).toBe(agent);
+ },
+ };
+ const agent = makeAgent([], { hooks: [hook] });
+ const r = await agent.systemOne("hi", QUESTIONS);
+ expect(events).toEqual(["start", "end"]);
+ expect(r.answers).toHaveProperty("q");
+ });
+
+ it("a start hook may rewrite the state the model sees", async () => {
+ const seen: string[] = [];
+ const agent = makeAgent(seen);
+ await agent.systemOne("original text", QUESTIONS, {
+ onPredictStart(ctx) {
+ ctx.states = ["rewritten text"];
+ },
+ });
+ expect(seen.join(" ")).toContain("rewritten text");
+ expect(seen.join(" ")).not.toContain("original text");
+ });
+
+ it("ctx.skip() short-circuits inference; end hooks still run", async () => {
+ const seen: string[] = [];
+ const events: string[] = [];
+ const cached = { model: "cache", answers: { q: { type: "noul", noul: 1 } }, usage: { input_tokens: 0, output_tokens: 0 } };
+ const agent = makeAgent(seen, {
+ onPredictStart(ctx) {
+ events.push("start");
+ ctx.skip([cached]);
+ },
+ onPredictEnd(ctx) {
+ events.push("end");
+ expect(ctx.usage?.input_tokens).toBe(0);
+ },
+ });
+ const r = await agent.systemOne("hi", QUESTIONS);
+ expect(events).toEqual(["start", "end"]);
+ expect(seen).toEqual([]); // the tokenizer never ran
+ expect(r).toEqual(cached);
+ });
+
+ it("an end hook may rewrite the results", async () => {
+ const agent = makeAgent([], {
+ onPredictEnd(ctx) {
+ for (const r of ctx.results ?? []) r["stamped"] = true;
+ },
+ });
+ const r = (await agent.systemOne("hi", QUESTIONS)) as Record<string, unknown>;
+ expect(r["stamped"]).toBe(true);
+ });
+
+ it("per-call hooks run after installed hooks", async () => {
+ const order: string[] = [];
+ const agent = makeAgent([], { hooks: [{ onPredictStart: () => order.push("installed") }] });
+ await agent.systemOne("hi", QUESTIONS, { onPredictStart: () => order.push("per-call") });
+ expect(order).toEqual(["installed", "per-call"]);
+ });
+
+ it("hooksRaise=false warns and continues when a hook fails", async () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const agent = makeAgent([], {
+ hooksRaise: false,
+ onPredictStart() {
+ throw new Error("telemetry down");
+ },
+ });
+ const r = await agent.systemOne("hi", QUESTIONS);
+ expect(r.answers).toHaveProperty("q");
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining("telemetry down"));
+ warn.mockRestore();
+ });
+
+ it("onError fires on inference failure; the original error propagates and end hooks still run", async () => {
+ const events: string[] = [];
+ const agent = makeAgent([], {
+ hooks: [
+ {
+ onError(ctx) {
+ events.push("error");
+ expect(String(ctx.error)).toContain("question");
+ },
+ onPredictEnd() {
+ events.push("end");
+ },
+ },
+ ],
+ });
+ await expect(agent.systemOne("hi", { bad: { type: "nope" } } as never)).rejects.toThrow(
+ 'question "bad"',
+ );
+ expect(events).toEqual(["error", "end"]);
+ });
+
+ it("a failing onError hook does not mask the original failure", async () => {
+ const agent = makeAgent([], {
+ hooks: [
+ {
+ onError() {
+ throw new Error("hook broke too");
+ },
+ },
+ ],
+ });
+ await expect(agent.systemOne("hi", { bad: { type: "nope" } } as never)).rejects.toThrow(
+ 'question "bad"',
+ );
+ });
+
+ it("a failing end hook propagates on the success path", async () => {
+ const agent = makeAgent([], {
+ onPredictEnd() {
+ throw new Error("end hook broke");
+ },
+ });
+ await expect(agent.systemOne("hi", QUESTIONS)).rejects.toThrow("end hook broke");
+ });
+});
+
+describe("router hooks", () => {
+ it("onRoute observes and may replace the decision", async () => {
+ const seen: string[] = [];
+ const { router, calls } = makeRouter({
+ hooks: [
+ {
+ onRoute(ctx: PredictContext) {
+ seen.push(ctx.decision?.model as string);
+ ctx.decision = {
+ model: "multilingual",
+ repo: "convaiinnovations/laya/multilingual",
+ reason: "pinned by a hook",
+ detection: null,
+ workflow: null,
+ };
+ },
+ },
+ ],
+ });
+ const d = router.route("hello there", QUESTIONS);
+ expect(seen).toEqual(["english"]); // detection ran before the hook replaced it
+ expect(d.model).toBe("multilingual");
+ const r = await router.predict("hello there", QUESTIONS);
+ expect(r.routing.model).toBe("multilingual");
+ expect(calls).toEqual(["hello there"]);
+ });
+
+ it("predict wraps route+infer: start hooks see ctx.decision, end hooks see results and usage", async () => {
+ let startCtx: PredictContext | null = null;
+ const { router } = makeRouter({
+ onPredictStart(ctx: PredictContext) {
+ startCtx = ctx;
+ expect(ctx.decision?.model).toBe("english");
+ expect(ctx.model).toBe("english");
+ expect(ctx.router).toBe(router);
+ },
+ onPredictEnd(ctx: PredictContext) {
+ expect(ctx.runId).toBe((startCtx as PredictContext | null)?.runId);
+ expect(ctx.results?.[0]).toHaveProperty("routing");
+ expect(ctx.usage).toEqual({ input_tokens: 5, output_tokens: 0 });
+ },
+ });
+ const r = await router.predict("plain english text", QUESTIONS);
+ expect(r.routing.model).toBe("english");
+ });
+
+ it("a cache hit still gets a routing key, without overwriting an existing one", async () => {
+ const { router, calls } = makeRouter({
+ onPredictStart(ctx: PredictContext) {
+ ctx.skip([{ model: "cache", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } }]);
+ },
+ });
+ const r = await router.predict("plain english text", QUESTIONS);
+ expect(calls).toEqual([]); // inference skipped
+ expect(r.routing.model).toBe("english");
+
+ const { router: router2 } = makeRouter({
+ onPredictStart(ctx: PredictContext) {
+ ctx.skip([
+ {
+ model: "cache",
+ answers: {},
+ usage: { input_tokens: 0, output_tokens: 0 },
+ routing: { model: "typed-decisions" },
+ },
+ ]);
+ },
+ });
+ const r2 = await router2.predict("plain english text", QUESTIONS);
+ expect(r2.routing.model).toBe("typed-decisions"); // cached routing wins
+ });
+
+ it("onLoad fires on the first load only; onEvict fires for the LRU victim", async () => {
+ const events: string[] = [];
+ const { router } = makeRouter({
+ maxLoaded: 1,
+ hooks: [
+ {
+ onLoad: (ctx: PredictContext) => events.push(`load:${String(ctx.model)}`),
+ onEvict: (ctx: PredictContext) => events.push(`evict:${String(ctx.model)}`),
+ },
+ ],
+ });
+ await router.load("english");
+ await router.load("english"); // cache hit: no event
+ await router.load("multilingual"); // exceeds maxLoaded=1
+ expect(events).toEqual(["load:english", "evict:english", "load:multilingual"]);
+ expect(router.loaded).toEqual(["multilingual"]);
+ });
+});
+
+describe("hook registry and normalisation", () => {
+ it("a plain callable as hooks is a TypeError; as onPredictStart it adapts", () => {
+ const fn = () => {};
+ expect(() => normaliseHooks(fn as never)).toThrow(/ambiguous/);
+ const out = normaliseHooks(undefined, fn);
+ expect(out).toHaveLength(1);
+ expect(typeof out[0].onPredictStart).toBe("function");
+ });
+
+ it("rejects the same callable for both start and end", () => {
+ const fn = () => {};
+ expect(() => normaliseHooks(undefined, fn, fn)).toThrow(/different callables/);
+ });
+
+ it("rejects objects with no hook methods and non-callable onPredictStart", () => {
+ expect(() => normaliseHooks({ notAHook: 1 } as never)).toThrow(/at least one hook method/);
+ expect(() => normaliseHooks(undefined, 42 as never)).toThrow(/onPredictStart must be callable/);
+ });
+
+ it("addHook/removeHook chain and withHooks scopes async callbacks", async () => {
+ const reg = new HookRegistry();
+ const hits: string[] = [];
+ const hook: Hook = { onPredictStart: () => hits.push("hit") };
+ expect(reg.addHook(hook)).toBe(reg);
+ expect(reg.hooks).toHaveLength(1);
+ expect(reg.removeHook(hook)).toBe(true);
+ expect(reg.removeHook(hook)).toBe(false);
+
+ const { router } = makeRouter();
+ await router.withHooks([hook], async () => {
+ expect(router.hooks).toHaveLength(1);
+ await router.predict("plain english text", QUESTIONS);
+ });
+ expect(hits).toEqual(["hit"]);
+ expect(router.hooks).toHaveLength(0); // removed after the callback settles
+ });
+});