Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

30.8 KB
Raw Blame
import { describe, expect, it, vi } from "vitest";
import { Agent, Router, clearDefaultHooks, setDefaultHooks } from "../src/index.js";
import type { BatchRequest } from "../src/index.js";

// TS mirror of tests/test_router_batch.py and the orchestration part of tests/test_batch.py.

const Q = { intent: { type: "noul", instructions: "Relevant?" } } as const;

// A stateful default hook must not run again inside the Agent after the Router has
// already expanded its questions (as OptionStabilityHook does).
it.each([false, true])("runs process defaults once per request with fallback=%s", async (fallback) => {
  const starts: unknown[] = [];
  const ends: unknown[] = [];
  const agent = new Agent({ provider: {
    async runEncoder(batch) { return { lastHidden: batch.inputIds.map(() => [[0]]) }; },
    async runHead(_hidden, batch) {
      return { logits: batch.qtype.map(() => [2, 0]), act: batch.qtype.map(() => [3, 0]) };
    },
  } });
  const loaded = fallback ? { systemOne: agent.systemOne.bind(agent) } : agent;
  const router = new Router({ default: "english", loader: (() => loaded) as never });
  setDefaultHooks({
    onPredictStart(ctx) { starts.push([...ctx.states]); },
    onPredictEnd(ctx) { ends.push([...ctx.states]); },
  });
  try {
    const results = await router.predictBatch([
      { state: "first request", questions: Q },
      { state: "second request", questions: Q },
    ]);
    expect(results).toHaveLength(2);
    expect(starts).toEqual([["first request"], ["second request"]]);
    // Ends unwrap in reverse start order: every request started before any ended.
    expect(ends).toEqual([...starts].reverse());
  } finally {
    clearDefaultHooks();
  }
});
const req = (state: unknown, overrides: Record<string, unknown> = {}): BatchRequest =>
  ({ state, questions: Q, ...overrides }) as unknown as BatchRequest;

const TYPED_KEYS = ["action", "needs_review", "outcome", "risk", "urgency"];
const typedQuestions = Object.fromEntries(
  TYPED_KEYS.map((k) => [k, { type: "noul", instructions: "?" }]),
);

// Fake checkpoint loader mirroring the Python fake_agent fixture: records which checkpoint
// was built and every predictBatch call, and fails when a state says "raise".
const makeRouter = (opts: Record<string, unknown> = {}) => {
  const built: string[] = [];
  const calls: [string, unknown[], Record<string, unknown>, unknown][] = [];
  const loader = (name: string, spec: { subfolder?: string | null }) => {
    const checkpoint = spec.subfolder ?? "english";
    built.push(checkpoint);
    return {
      checkpoint,
      async predictBatch(states: unknown[], questions: Record<string, unknown>, batchOpts?: unknown) {
        calls.push([checkpoint, [...states], questions, batchOpts ?? null]);
        if (states.includes("raise")) throw new Error("inference failed");
        return states.map((state) => ({
          model: "laya-rl-agent",
          answers: { seen: state },
          usage: {},
        }));
      },
      async systemOne(state: unknown, questions: Record<string, unknown>) {
        return (await this.predictBatch([state], questions))[0];
      },
    };
  };
  const router = new Router({ default: "english", loader: loader as never, ...opts });
  return { router, built, calls };
};

const mixedItems = (): BatchRequest[] => [
  req("English text one"),
  req("\u0645\u0631\u062d\u0628\u0627"),
  req("English text two"),
  { state: "decision", questions: typedQuestions } as unknown as BatchRequest,
  req("forced", { model: "ml" }),
  req("forced english", { lang: "en" }),
  req("explicit task", { task: "typed_decisions" }),
];

describe("Router.routeBatch / predictBatch", () => {
  it("mixed groups keep input order and share one batch per checkpoint", async () => {
    const { router, built, calls } = makeRouter({ maxLoaded: 3, autoTaskDetection: true });
    const items = mixedItems();
    const decisions = router.routeBatch(items);
    expect(router.loaded).toEqual([]); // routeBatch routes without loading
    const results = await router.predictBatch(items);
    expect(results.map((r) => r.answers.seen)).toEqual(items.map((r) => r.state));
    expect(results.map((r) => r.routing)).toEqual(decisions);
    expect(built).toEqual(["english", "multilingual", "typed-decisions"]);
    expect(calls.map(([c, s]) => [c, s])).toEqual([
      ["english", ["English text one", "English text two", "forced english"]],
      ["multilingual", ["\u0645\u0631\u062d\u0628\u0627", "forced"]],
      ["typed-decisions", ["decision"]],
      ["typed-decisions", ["explicit task"]], // different schema: its own batch
    ]);
    expect(await router.predictMany([])).toEqual([]);
    expect(router.routeBatch([])).toEqual([]);
  });

  it("evicts least-recently-used checkpoints under capacity pressure", async () => {
    const { router, built } = makeRouter({ maxLoaded: 1, autoTaskDetection: true });
    await router.predictBatch(mixedItems());
    expect(built).toEqual(["english", "multilingual", "typed-decisions"]);
    expect(router.loaded).toEqual(["typed-decisions"]);
  });

  it("rejects malformed batches before loading anything", () => {
    const { router, built } = makeRouter();
    expect(() => router.routeBatch(null as never)).toThrow(TypeError);
    expect(() => router.routeBatch("text" as never)).toThrow(/requests must be an array/);
    expect(() => router.routeBatch([null] as never)).toThrow(/request 0/);
    expect(() => router.routeBatch([{ questions: Q }] as never)).toThrow(
      /request 0 is missing required key 'state'/,
    );
    expect(() => router.routeBatch([{ state: null, questions: Q }] as never)).toThrow(
      /request 0 'state' must not be None/,
    );
    expect(() => router.routeBatch([{ state: "x" }] as never)).toThrow(
      /request 0 is missing required key 'questions'/,
    );
    expect(() => router.routeBatch([req("x"), { state: "y", questions: null }] as never)).toThrow(
      /request 1 'questions'/,
    );
    expect(() => router.routeBatch([req("x"), req("y", { model: "invalid" })])).toThrow(
      /unknown model/,
    );
    expect(built).toEqual([]);
  });

  it("propagates inference failures and the router stays consistent afterwards", async () => {
    const { router, calls } = makeRouter({ maxLoaded: 2 });
    await expect(
      router.predictBatch([req("first"), req("raise", { lang: "ar" }), req("unreached", { lang: "ar" })]),
    ).rejects.toThrow("inference failed");
    expect(calls.map(([c, s]) => [c, s])).toEqual([
      ["english", ["first"]],
      ["multilingual", ["raise", "unreached"]], // one shared batch, even though it fails
    ]);
    const after = await router.predict("after failure", Q as never, { lang: "ar" });
    expect(after.answers.seen).toBe("after failure");
  });

  it("sends same-checkpoint same-schema requests in one agent batch, passing batchSize", async () => {
    const { router, calls } = makeRouter({ maxLoaded: 2 });
    const results = await router.predictBatch(
      [req("one", { model: "english" }), req("two", { model: "english" }), req("three", { model: "english" })],
      2,
    );
    expect(results.map((r) => r.answers.seen)).toEqual(["one", "two", "three"]);
    expect(calls.length).toBe(1);
    expect(calls[0][0]).toBe("english");
    expect(calls[0][1]).toEqual(["one", "two", "three"]);
    expect(calls[0][2]).toEqual(Q);
    expect(calls[0][3]).toEqual({ batchSize: 2 });
  });

  it("splits same-checkpoint requests with different question schemas", async () => {
    const { router, calls } = makeRouter({ maxLoaded: 2 });
    const q2 = { risk: { type: "noul", instructions: "Risky?" } };
    const results = await router.predictBatch([
      { state: "one", questions: Q, model: "english" } as unknown as BatchRequest,
      { state: "two", questions: q2, model: "english" } as unknown as BatchRequest,
      { state: "three", questions: Q, model: "english" } as unknown as BatchRequest,
    ]);
    expect(results.map((r) => r.answers.seen)).toEqual(["one", "two", "three"]);
    expect(calls.length).toBe(2);
    expect(calls[0][1]).toEqual(["one", "three"]);
    expect(calls[0][2]).toEqual(Q);
    expect(calls[1][1]).toEqual(["two"]);
    expect(calls[1][2]).toEqual(q2);
  });

  it("routeBatch forwards per-request langGuess overrides", () => {
    const { router, built } = makeRouter();
    const decisions = router.routeBatch([
      req("hola", { lang_guess: "es" }),
      req("hello", { langGuess: "en-US" }),
    ]);
    expect(decisions.map((d) => d.model)).toEqual(["multilingual", "english"]);
    expect(built).toEqual([]);
  });

  // A question schema that arrives with a different key order must be scored with its own
  // order: options are positional in the rendered sequence, so grouping reordered-but-equal
  // schemas would make the second request's batched answers differ from its single-request
  // answers. (TS mirror of the #166 regression test.)
  it("scores reordered-but-equal question schemas separately", async () => {
    const ordered = {
      intent: { type: "choice", instructions: "Pick one", criteria: { zulu: "last", alpha: "first" } },
    };
    const reordered = {
      intent: { type: "choice", instructions: "Pick one", criteria: { alpha: "first", zulu: "last" } },
    };
    const { router, calls } = makeRouter({ maxLoaded: 1 });
    const results = await router.predictBatch([
      { state: "one", questions: ordered } as unknown as BatchRequest,
      { state: "two", questions: reordered } as unknown as BatchRequest,
    ]);
    expect(results.length).toBe(2);
    expect(calls.map((c) => Object.keys((c[2] as never as typeof ordered).intent.criteria))).toEqual([
      ["zulu", "alpha"],
      ["alpha", "zulu"],
    ]);
  });

  it("accepts per-call hooks in predict order, and treats null and [] as no-ops", async () => {
    const order: string[] = [];
    const installed = {
      onRoute(ctx: { states: unknown[] }) {
        order.push(`installed:route:${ctx.states[0]}`);
      },
      onPredictStart(ctx: { states: unknown[] }) {
        order.push(`installed:start:${ctx.states[0]}`);
      },
      onPredictEnd(ctx: { states: unknown[] }) {
        order.push(`installed:end:${ctx.states[0]}`);
      },
    };
    const perCall = {
      onRoute(ctx: { states: unknown[] }) {
        order.push(`percall:route:${ctx.states[0]}`);
      },
      onPredictStart(ctx: { states: unknown[] }) {
        order.push(`percall:start:${ctx.states[0]}`);
      },
      onPredictEnd(ctx: { states: unknown[] }) {
        order.push(`percall:end:${ctx.states[0]}`);
      },
    };
    const { router } = makeRouter({ hooks: [installed] });
    const items = [req("one", { model: "english" }), req("two", { model: "english" })];
    const results = await router.predictBatch(items, null, { hooks: [perCall] });
    expect(results.map((r) => r.answers.seen)).toEqual(["one", "two"]);
    // routeBatch runs before any predict hook. End hooks unwind in REVERSE start order (the
    // LIFO rule Python's Router.predict_batch documents), and within one request the installed
    // hook still runs before the per-call one.
    expect(order).toEqual([
      "installed:route:one",
      "percall:route:one",
      "installed:route:two",
      "percall:route:two",
      "installed:start:one",
      "percall:start:one",
      "installed:start:two",
      "percall:start:two",
      "installed:end:two",
      "percall:end:two",
      "installed:end:one",
      "percall:end:one",
    ]);

    order.length = 0;
    await router.predictBatch(items, null, { hooks: null });
    const noneOrder = [...order];
    order.length = 0;
    await router.predictBatch(items, null, { hooks: [] });
    const emptyOrder = [...order];
    order.length = 0;
    await router.predictBatch(items);
    expect(noneOrder).toEqual(emptyOrder);
    expect(emptyOrder).toEqual(order);
    expect(noneOrder).toEqual([
      "installed:route:one",
      "installed:route:two",
      "installed:start:one",
      "installed:start:two",
      "installed:end:two",
      "installed:end:one",
    ]);

    const { router: bare } = makeRouter();
    expect((await bare.predictBatch(items, null, { hooks: null })).map((r) => r.answers.seen)).toEqual([
      "one",
      "two",
    ]);
    expect((await bare.predictBatch(items, null, { hooks: [] })).map((r) => r.answers.seen)).toEqual([
      "one",
      "two",
    ]);
  });

  it("runs per-call onPredictStart and onPredictEnd once per request", async () => {
    const events: string[] = [];
    const { router, calls } = makeRouter();
    const items = [req("one", { model: "english" }), req("two", { model: "english" })];
    const results = await router.predictBatch(items, null, {
      onPredictStart(ctx) {
        events.push(`start:${ctx.states[0]}`);
      },
      onPredictEnd(ctx) {
        events.push(`end:${ctx.states[0]}`);
      },
    });
    expect(results.map((r) => r.answers.seen)).toEqual(["one", "two"]);
    expect(calls.length).toBe(1);
    // Starts run for every request in the checkpoint group, then ends, in REVERSE start
    // order -- the LIFO unwind Python's Router.predict_batch documents.
    expect(events).toEqual(["start:one", "start:two", "end:two", "end:one"]);
  });

  it("ends three request contexts in reverse start order", async () => {
    // Two requests cannot tell FIFO from LIFO apart in a way that names the rule, and this is
    // the shape a tracing hook depends on: start a, b, c then end c, b, a.
    const events: string[] = [];
    const { router } = makeRouter();
    await router.predictBatch(
      [req("a", { model: "english" }), req("b", { model: "english" }), req("c", { model: "english" })],
      null,
      {
        onPredictStart(ctx) {
          events.push(`start:${ctx.states[0]}`);
        },
        onPredictEnd(ctx) {
          events.push(`end:${ctx.states[0]}`);
        },
      },
    );
    expect(events).toEqual([
      "start:a", "start:b", "start:c",
      "end:c", "end:b", "end:a",
    ]);
  });

  it("hooksRaise false swallows a throwing per-call hook; the default still raises", async () => {
    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
    try {
      const items = [req("one", { model: "english" }), req("two", { model: "english" })];
      const { router } = makeRouter();
      await expect(
        router.predictBatch(items, null, {
          onPredictStart() {
            throw new Error("hook failed");
          },
        }),
      ).rejects.toThrow("hook failed");

      const { router: quietByDefault } = makeRouter({ hooksRaise: false });
      const quiet = await quietByDefault.predictBatch(items, null, {
        onPredictStart() {
          throw new Error("installed policy");
        },
      });
      expect(quiet.map((r) => r.answers.seen)).toEqual(["one", "two"]);

      const seen: string[] = [];
      const { router: perCall } = makeRouter();
      const results = await perCall.predictBatch(items, null, {
        hooksRaise: false,
        onPredictStart(ctx) {
          seen.push(String(ctx.states[0]));
          throw new Error("per-call policy");
        },
      });
      expect(seen).toEqual(["one", "two"]);
      expect(results.map((r) => r.answers.seen)).toEqual(["one", "two"]);
      expect(warn.mock.calls.some((args) => String(args[0]).includes("per-call policy"))).toBe(true);
    } finally {
      warn.mockRestore();
    }
  });

  // Two gaps the cases above leave open, both named when #943 was closed in favour of #936.
  it("lets a per-call hooksRaise turn raising on where the router has it off", async () => {
    const throwing = {
      onPredictEnd() {
        throw new Error("hook blew up");
      },
    };
    const quiet = makeRouter({ hooks: [throwing], hooksRaise: false });
    // Router-level off, nothing per-call: the hook throws and the call still succeeds.
    expect((await quiet.router.predictBatch([req("s", { model: "english" })])).length).toBe(1);

    const loud = makeRouter({ hooks: [throwing], hooksRaise: false });
    // The override only means something if it is honoured here: without it this resolves.
    await expect(
      loud.router.predictBatch([req("s", { model: "english" })], null, { hooksRaise: true }),
    ).rejects.toThrow("hook blew up");
  });

  it("appends per-call callbacks after per-call hooks, the composition predict uses", async () => {
    // Order, not membership: both entries run under either composition, so only the sequence
    // fails if the arguments to composeHooks are swapped.
    const order: string[] = [];
    const { router } = makeRouter({});
    await router.predictBatch([req("s", { model: "english" })], null, {
      hooks: [{
        onPredictStart() {
          order.push("hooks");
        },
      }],
      onPredictStart() {
        order.push("onPredictStart");
      },
    });
    expect(order).toEqual(["hooks", "onPredictStart"]);
  });

  it("predictMany forwards onPredictStart, onPredictEnd and hooksRaise", async () => {
    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
    try {
      const events: string[] = [];
      const { router } = makeRouter();
      const items = [req("one", { model: "english" }), req("two", { model: "english" })];
      const results = await router.predictMany(items, null, {
        hooksRaise: false,
        onPredictStart(ctx) {
          events.push(`start:${ctx.states[0]}`);
          if (ctx.states[0] === "one") throw new Error("start failed");
        },
        onPredictEnd(ctx) {
          events.push(`end:${ctx.states[0]}`);
        },
      });
      expect(results.map((r) => r.answers.seen)).toEqual(["one", "two"]);
      expect(events).toEqual(["start:one", "start:two", "end:two", "end:one"]);
    } finally {
      warn.mockRestore();
    }
  });

  it("runs router hooks per request and short-circuits skipped requests", async () => {
    const events: string[] = [];
    const { router, calls } = makeRouter({
      maxLoaded: 2,
      hooks: [{
        onPredictStart(ctx: never) {
          const c = ctx as { states: unknown[]; decision: { model: string }; skip: (r: unknown[]) => void };
          events.push(`start:${c.states[0]}:${c.decision.model}`);
          if (c.states[0] === "hit") {
            c.skip([{ model: "cached", answers: { seen: "cached" }, usage: {} }]);
          }
        },
        onPredictEnd(ctx: never) {
          const c = ctx as { states: unknown[]; results: { answers: { seen: unknown } }[] | null;
            usage: unknown; elapsedMs: number | null };
          events.push(`end:${c.states[0]}:${c.results?.[0]?.answers.seen}`);
          expect(c.usage).not.toBeNull(); // stamped before any end hook runs
          expect(c.elapsedMs).not.toBeNull();
        },
      }],
    });
    const results = await router.predictBatch([
      req("miss", { model: "english" }),
      req("hit", { model: "english" }),
      req("miss2", { model: "english" }),
    ]);
    expect(results.map((r) => r.answers.seen)).toEqual(["miss", "cached", "miss2"]);
    expect(calls.length).toBe(1); // the skipped request shared no forward pass
    expect(calls[0][1]).toEqual(["miss", "miss2"]);
    expect(events.filter((e) => e.startsWith("start:")).length).toBe(3);
    expect(events.filter((e) => e.startsWith("end:")).length).toBe(3);
    // predict still promises a routing key on a cache hit
    expect(results[1].routing.model).toBe("english");
  });

  it("fires onError per started request and end hooks even on failure", async () => {
    const errors: string[] = [];
    const ends: unknown[] = [];
    const { router } = makeRouter({
      maxLoaded: 2,
      hooks: [{
        onError(ctx: never) {
          const c = ctx as { states: unknown[]; error: Error };
          errors.push(`${c.states[0]}:${c.error.message}`);
        },
        onPredictEnd(ctx: never) {
          ends.push((ctx as { states: unknown[] }).states[0]);
        },
      }],
    });
    await expect(
      router.predictBatch([req("first"), req("raise", { lang: "ar" }), req("unreached", { lang: "ar" })]),
    ).rejects.toThrow("inference failed");
    // only the failed group's started-without-result requests see onError
    expect(errors).toEqual(["raise:inference failed", "unreached:inference failed"]);
    expect([...ends].sort()).toEqual(["first", "raise", "unreached"]);
  });

  it("splits groups when start hooks set different token-budget overrides", async () => {
    const { router, calls } = makeRouter({
      maxLoaded: 2,
      hooks: [{
        onPredictStart(ctx: never) {
          const c = ctx as { states: unknown[]; maxLen: number | null };
          if (c.states[0] === "two") c.maxLen = 64;
        },
      }],
    });
    await router.predictBatch([req("one", { model: "english" }), req("two", { model: "english" })]);
    expect(calls.map((c) => c[1])).toEqual([["one"], ["two"]]);
    // overrides are only passed when set, so Agent-like objects without them still work
    expect(calls[0][3]).toEqual({ batchSize: null });
    expect(calls[1][3]).toEqual({ batchSize: null, maxLen: 64 });
  });

  it("seeds request budgets before hooks and groups by both final budgets", async () => {
    const { router, calls } = makeRouter();
    const seen: unknown[] = [];
    const results = await router.predictBatch([
      req("a", { model: "english", maxLen: 64, headMaxLen: 16 }),
      req("b", { model: "english", maxLen: 128, headMaxLen: 16 }),
      req("c", { model: "english", maxLen: 64, headMaxLen: 32 }),
      req("d", { model: "english", maxLen: 64, headMaxLen: 16 }),
    ], 2, {
      onPredictStart(ctx) { seen.push([ctx.maxLen, ctx.headMaxLen]); },
    });
    expect(seen).toEqual([[64, 16], [128, 16], [64, 32], [64, 16]]);
    expect(calls.map((c) => [c[1], c[3]])).toEqual([
      [["a", "d"], { batchSize: 2, maxLen: 64, headMaxLen: 16 }],
      [["b"], { batchSize: 2, maxLen: 128, headMaxLen: 16 }],
      [["c"], { batchSize: 2, maxLen: 64, headMaxLen: 32 }],
    ]);
    expect(results.map((r) => r.answers.seen)).toEqual(["a", "b", "c", "d"]);
  });

  it("lets hooks replace or clear explicit budgets before grouping", async () => {
    const { router, calls } = makeRouter();
    const seen: unknown[] = [];
    await router.predictBatch([
      req("replace", { model: "english", maxLen: 64, headMaxLen: 16 }),
      req("clear", { model: "english", maxLen: 128, headMaxLen: 32 }),
      req("defaults", { model: "english" }),
    ], null, {
      onPredictStart(ctx) {
        seen.push([ctx.maxLen, ctx.headMaxLen]);
        if (ctx.states[0] === "replace") {
          ctx.maxLen = 96;
          ctx.headMaxLen = 24;
        } else if (ctx.states[0] === "clear") {
          ctx.maxLen = ctx.headMaxLen = null;
        }
      },
    });
    expect(seen).toEqual([[64, 16], [128, 32], [null, null]]);
    expect(calls.map((c) => [c[1], c[3]])).toEqual([
      [["replace"], { batchSize: null, maxLen: 96, headMaxLen: 24 }],
      [["clear", "defaults"], { batchSize: null }],
    ]);
  });

  it("keeps absent and null budgets unset without dropping zero overrides", async () => {
    const { router, calls } = makeRouter();
    await router.predictBatch([
      req("absent", { model: "english" }),
      req("null", { model: "english", maxLen: null, headMaxLen: null }),
      req("zero", { model: "english", maxLen: 0, headMaxLen: 0 }),
    ]);
    expect(calls.map((c) => [c[1], c[3]])).toEqual([
      [["absent", "null"], { batchSize: null }],
      [["zero"], { batchSize: null, maxLen: 0, headMaxLen: 0 }],
    ]);
  });

  it("raises when the agent returns the wrong number of results", async () => {
    const router = new Router({
      loader: (() => ({
        async predictBatch() { return []; },
        async systemOne() { return {}; },
      })) as never,
    });
    await expect(
      router.predictBatch([req("a", { model: "english" }), req("b", { model: "english" })]),
    ).rejects.toThrow(/returned 0 results for 2 states/);
  });

  it.each([false, true])("forwards fallback budgets with hook override=%s", async (override) => {
    const calls: unknown[] = [];
    const router = new Router({
      loader: (() => ({
        langTemperatures: { en: {} },
        async systemOne(state: unknown, _questions: unknown, opts: unknown) {
          calls.push([state, opts]);
          return { model: "stub", answers: { seen: state }, usage: {} };
        },
      })) as never,
    });
    const results = await router.predictBatch([
      req("budgeted", { model: "english", lang: "en", maxLen: 64, headMaxLen: 16 }),
      req("defaults", { model: "english", lang: "en", maxLen: null, headMaxLen: null }),
      req("zero", { model: "english", lang: "en", maxLen: 0, headMaxLen: 0 }),
    ], 2, {
      onPredictStart(ctx) {
        if (override && ctx.states[0] === "budgeted") {
          ctx.maxLen = 96;
          ctx.headMaxLen = 24;
        }
      },
    });
    expect(calls).toEqual([
      ["budgeted", { lang: "en", maxLen: override ? 96 : 64, headMaxLen: override ? 24 : 16 }],
      ["defaults", { lang: "en" }],
      ["zero", { lang: "en", maxLen: 0, headMaxLen: 0 }],
    ]);
    expect(results.map((r) => r.answers.seen)).toEqual(["budgeted", "defaults", "zero"]);
  });

  it("falls back to systemOne for agent-like objects without predictBatch", async () => {
    const router = new Router({
      loader: (() => ({
        async systemOne(state: unknown) {
          return { model: "stub", answers: { seen: state }, usage: {} };
        },
      })) as never,
    });
    const results = await router.predictBatch([req("a", { model: "english" }), req("b", { model: "english" })]);
    expect(results.map((r) => r.answers.seen)).toEqual(["a", "b"]);
    expect(results.every((r) => r.routing.model === "english")).toBe(true);
  });
});

// Agent-level batching, mirroring the orchestration tests in tests/test_batch.py: states
// sharing one question schema are collated into one forward pass per batchSize chunk.
const tokenizer = () => ({
  clsId: 1,
  sepId: 2,
  maskId: 3,
  padId: 0,
  maskToken: "[MASK]",
  encode(text: string): number[] {
    return Array.from(text).map((c) => c.codePointAt(0) ?? 0);
  },
});

const QUESTIONS = {
  flag: { type: "noul", instructions: "Relevant?" },
  pick: { type: "choice", instructions: "Pick", criteria: { a: "A", b: "B" } },
} as never;

// Row r gets logits [r + 1, 0], so each state's noul value reveals which row it decoded.
const makeCountingAgent = () => {
  const encodeCalls: number[] = [];
  const provider = {
    async runEncoder(batch: { inputIds: number[][] }) {
      encodeCalls.push(batch.inputIds.length);
      return { lastHidden: batch.inputIds.map(() => [[0, 0]]) };
    },
    async runHead(_hidden: unknown, batch: { inputIds: number[][] }) {
      const n = batch.inputIds.length;
      return {
        logits: Array.from({ length: n }, (_, r) => [r + 1, 0]),
        act: Array.from({ length: n }, () => [1, 0]),
      };
    },
  };
  const agent = new Agent({ provider, tok: tokenizer() } as never);
  return { agent, encodeCalls };
};

const expectedNoul = (row: number): number =>
  Math.round((1 / (Math.exp(row + 1) + 1)) * 1e4) / 1e4;

describe("Agent.predictBatch", () => {
  it("packs all states into one forward pass by default, in input order", async () => {
    const { agent, encodeCalls } = makeCountingAgent();
    const results = await agent.predictBatch(["one", "two", "three"], QUESTIONS);
    expect(encodeCalls).toEqual([6]); // 3 states x 2 questions, one pass
    expect(results.length).toBe(3);
    // state s decodes from rows 2s (flag) and 2s+1 (pick)
    expect(results.map((r) => (r.answers.flag as { noul: number }).noul)).toEqual([
      expectedNoul(0),
      expectedNoul(2),
      expectedNoul(4),
    ]);
  });

  it("batchSize chunks states across forward passes", async () => {
    const { agent, encodeCalls } = makeCountingAgent();
    const results = await agent.predictBatch(["1", "2", "3", "4", "5"], QUESTIONS, { batchSize: 2 });
    expect(encodeCalls).toEqual([4, 4, 2]); // 2+2+1 states x 2 questions
    expect(results.length).toBe(5);
    expect(results.map((r) => (r.answers.flag as { noul: number }).noul)).toEqual([
      expectedNoul(0), expectedNoul(2), expectedNoul(0), expectedNoul(2), expectedNoul(0),
    ]); // row offsets restart per chunk
  });

  it("handles empty states and empty questions without encoding", async () => {
    const { agent, encodeCalls } = makeCountingAgent();
    expect(await agent.predictBatch([], QUESTIONS)).toEqual([]);
    const [res] = await agent.predictBatch(["state"], {});
    expect(res.answers).toEqual({});
    expect(res.usage.input_tokens).toBe(0);
    expect(encodeCalls).toEqual([]);
  });

  it("matches systemOne output for a single state", async () => {
    const { agent } = makeCountingAgent();
    const batched = await agent.predictBatch(["state text"], QUESTIONS);
    const single = await agent.systemOne("state text", QUESTIONS);
    expect(batched[0]).toEqual(single);
  });

  it("reports per-state input token usage", async () => {
    const { agent } = makeCountingAgent();
    const results = await agent.predictBatch(["aaaa", "b"], QUESTIONS);
    expect(results[0].usage.input_tokens).toBeGreaterThan(results[1].usage.input_tokens);
    const solo = await agent.systemOne("aaaa", QUESTIONS);
    expect(results[0].usage.input_tokens).toBe(solo.usage.input_tokens);
  });

  it("supports skip from a start hook without encoding", async () => {
    const { agent, encodeCalls } = makeCountingAgent();
    const results = await agent.predictBatch(["a", "b"], QUESTIONS, {
      onPredictStart(ctx: never) {
        (ctx as { skip: (r: unknown[]) => void }).skip([
          { model: "cached", answers: {}, usage: { input_tokens: 0, output_tokens: 0 } },
        ]);
      },
    });
    expect(results[0].model).toBe("cached");
    expect(encodeCalls).toEqual([]);
  });

  it("router.decideBatch evaluates schema across batch", async () => {
    const customLoader = () => ({
      async predictBatch(states: unknown[]) {
        return states.map(() => ({
          model: "fake",
          answers: { flag: { type: "noul", noul: 0.1, confidence: 0.2, answer_confidence: 0.2 } },
        }));
      },
      async systemOne(state: unknown) {
        return {
          model: "fake",
          answers: { flag: { type: "noul", noul: 0.1, confidence: 0.2, answer_confidence: 0.2 } },
        };
      },
    });
    const router = new Router({ loader: customLoader as never });
    const schema = {
      type: "object",
      properties: {
        flag: { type: "boolean" },
      },
    };
    const res1 = await router.decideBatch(["alpha", "beta"], schema);
    expect(res1).toHaveLength(2);
    expect(res1[0].flag).toBe(false);

    const res2 = await router.decideBatch(["alpha"], schema, { returnDetails: true });
    expect(res2).toHaveLength(1);
    expect(res2[0].values).toBeDefined();
    expect(res2[0].answerConfidence).toBeDefined();
    expect(res2[0].confidence).toBeDefined();

    // Verify min_confidence alias gates low confidence on the router path
    const resGated = await router.decideBatch(["alpha"], schema, { min_confidence: 0.99 });
    expect(resGated).toHaveLength(1);
    expect(resGated[0].flag).toBeNull();

    // Verify minConfidence also gates low confidence on the router path
    const resGatedCamel = await router.decideBatch(["alpha"], schema, { minConfidence: 0.99 });
    expect(resGatedCamel).toHaveLength(1);
    expect(resGatedCamel[0].flag).toBeNull();
  });
});