Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

Add tool-stub rules, argument matching, and outcome sequences (1/3) (#4431)

Philipp Comans3aed44bb1bac
diff --git a/packages/eve/src/tool-stubs/playback.ts b/packages/eve/src/tool-stubs/playback.ts
new file mode 100644
index 000000000..377902d4e
--- /dev/null
+++ b/packages/eve/src/tool-stubs/playback.ts
@@ -0,0 +1,75 @@
+import { isObject } from "#shared/guards.js";
+import { createStubValidator } from "#tool-stubs/schema.js";
+import type { StubCall, StubResult, ToolStub } from "#tool-stubs/types.js";
+
+/**
+ * Selects stub outcomes and advances their sequences.
+ * The surrounding workflow orders calls and restores progress on replay.
+ */
+export class StubPlayback {
+ private readonly rules;
+ private readonly results = new Map<string, StubResult>();
+ private readonly positions = new Map<string, number>();
+
+ constructor(rules: readonly ToolStub[]) {
+ this.rules = rules.map((rule) => ({
+ ...rule,
+ constraints: Object.entries(rule.match ?? {}).map(([property, schema]) => {
+ try {
+ return { property, validator: createStubValidator(schema) };
+ } catch (cause) {
+ throw new Error(
+ `Could not create validator for matcher "${property}" in tool stub "${rule.id}".`,
+ {
+ cause,
+ },
+ );
+ }
+ }),
+ }));
+ }
+
+ call(call: StubCall): StubResult {
+ const recorded = this.results.get(call.callId);
+ if (recorded !== undefined) return recorded;
+ if (this.results.size >= 10_000)
+ return {
+ kind: "error",
+ error: "Tool stub session exceeded 10,000 calls. Start a new eval session.",
+ };
+ const candidates = this.rules.filter((rule) => rule.tool === call.tool);
+ if (call.persistent && candidates.some((rule) => rule.constraints.length > 0)) {
+ return this.record(call, {
+ kind: "error",
+ error: `Persistent tool "${call.tool}" requires an unconditional stub.`,
+ });
+ }
+ const rule = candidates.find((rule) =>
+ rule.constraints.every(
+ ({ property, validator }) =>
+ isObject(call.input) &&
+ Object.hasOwn(call.input, property) &&
+ validator.validate(call.input[property]).valid,
+ ),
+ );
+ if (rule === undefined) return this.record(call, { kind: "real" });
+ const outcomes = rule.outcomes ?? [rule.outcome!];
+ const position = Math.min(this.positions.get(rule.id) ?? 0, outcomes.length - 1);
+ this.positions.set(rule.id, position + 1);
+ return this.record(call, {
+ kind: "stub",
+ ruleId: rule.id,
+ position,
+ outcome: outcomes[position]!,
+ });
+ }
+
+ private record(call: StubCall, result: StubResult): StubResult {
+ this.results.set(call.callId, result);
+ return result;
+ }
+
+ fail(callId: string, error: string): StubResult {
+ return this.results.get(callId)?.kind === "stub" ? { kind: "error", error } : { kind: "real" };
+ }
+}
diff --git a/packages/eve/src/tool-stubs/rules.test.ts b/packages/eve/src/tool-stubs/rules.test.ts
new file mode 100644
index 000000000..ea730ea32
--- /dev/null
+++ b/packages/eve/src/tool-stubs/rules.test.ts
@@ -0,0 +1,347 @@
+import { describe, expect, it } from "vitest";
+
+import { parseToolStubs } from "#tool-stubs/rules.js";
+import { StubPlayback } from "#tool-stubs/playback.js";
+
+describe("tool stubs", () => {
+ it("rejects prototype keys instead of silently dropping a match constraint", () => {
+ expect(() =>
+ parseToolStubs(
+ JSON.parse(
+ '[{"id":"a","tool":"list","match":{"__proto__":{"const":"x"}},"outcome":{"response":null}}]',
+ ),
+ ),
+ ).toThrow(/prototype/i);
+ });
+
+ it("selects a response by partial input without changing the arguments", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ {
+ id: "milk",
+ tool: "complete_task",
+ match: { task_id: { const: "milk" } },
+ outcome: { response: { success: true } },
+ },
+ ]),
+ );
+ const input = { task_id: "milk", reason: "Done" };
+
+ expect(playback.call({ callId: "call-1", tool: "complete_task", input })).toEqual({
+ kind: "stub",
+ ruleId: "milk",
+ position: 0,
+ outcome: { response: { success: true } },
+ });
+ expect(input).toEqual({ task_id: "milk", reason: "Done" });
+ });
+
+ it("advances per call, repeats the last response, and reuses retried calls", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ {
+ id: "tasks",
+ tool: "list_tasks",
+ outcomes: [{ response: ["milk", "dog"] }, { response: ["dog"] }],
+ },
+ ]),
+ );
+ const call = { tool: "list_tasks", input: {} };
+ expect(playback.call({ ...call, callId: "first" })).toEqual({
+ kind: "stub",
+ ruleId: "tasks",
+ position: 0,
+ outcome: { response: ["milk", "dog"] },
+ });
+ expect(playback.call({ ...call, callId: "second" })).toEqual({
+ kind: "stub",
+ ruleId: "tasks",
+ position: 1,
+ outcome: { response: ["dog"] },
+ });
+ expect(playback.call({ ...call, callId: "first" })).toEqual({
+ kind: "stub",
+ ruleId: "tasks",
+ position: 0,
+ outcome: { response: ["milk", "dog"] },
+ });
+ expect(playback.call({ ...call, callId: "third" })).toEqual({
+ kind: "stub",
+ ruleId: "tasks",
+ position: 1,
+ outcome: { response: ["dog"] },
+ });
+ });
+
+ it("requires each named field and matches nested shapes without filling defaults", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ {
+ id: "search",
+ tool: "search",
+ match: {
+ filter: {
+ type: "object",
+ properties: { status: { const: "open", default: "open" } },
+ required: ["status"],
+ },
+ query: { type: "string", enum: ["milk", "buy milk"] },
+ tags: { type: "array", contains: { const: "urgent" } },
+ },
+ outcome: { response: ["milk"] },
+ },
+ ]),
+ );
+ expect(playback.call({ callId: "missing", tool: "search", input: {} })).toEqual({
+ kind: "real",
+ });
+ expect(
+ playback.call({
+ callId: "default",
+ tool: "search",
+ input: { filter: {}, query: "milk", tags: ["urgent"] },
+ }),
+ ).toEqual({ kind: "real" });
+ expect(
+ playback.call({
+ callId: "match",
+ tool: "search",
+ input: {
+ filter: { status: "open", owner: "alice" },
+ query: "buy milk",
+ tags: ["personal", "urgent"],
+ limit: 10,
+ },
+ }),
+ ).toEqual({ kind: "stub", ruleId: "search", position: 0, outcome: { response: ["milk"] } });
+ });
+
+ it("uses the first match, advances only its sequence, and never falls through after exhaustion", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ {
+ id: "specific",
+ tool: "list",
+ match: { status: { const: "open" } },
+ outcomes: [{ response: "one" }, { response: "two" }],
+ },
+ {
+ id: "fallback",
+ tool: "list",
+ outcomes: [{ response: "fallback-one" }, { response: "fallback-two" }],
+ },
+ ]),
+ );
+ const call = (callId: string, status: string) =>
+ playback.call({ callId, tool: "list", input: { status } });
+ expect(call("a", "open")).toMatchObject({
+ ruleId: "specific",
+ outcome: { response: "one" },
+ position: 0,
+ });
+ expect(call("b", "open")).toMatchObject({
+ ruleId: "specific",
+ outcome: { response: "two" },
+ position: 1,
+ });
+ expect(call("c", "open")).toMatchObject({
+ ruleId: "specific",
+ outcome: { response: "two" },
+ position: 1,
+ });
+ expect(call("a", "open")).toMatchObject({
+ ruleId: "specific",
+ outcome: { response: "one" },
+ position: 0,
+ });
+ expect(call("d", "closed")).toMatchObject({
+ ruleId: "fallback",
+ outcome: { response: "fallback-one" },
+ position: 0,
+ });
+ });
+
+ it("rejects conditional replacement of persistent tools", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ {
+ id: "agent",
+ tool: "tasks_agent",
+ match: { message: { const: "hello" } },
+ outcome: { response: "hi" },
+ },
+ ]),
+ );
+ expect(
+ playback.call({
+ callId: "call",
+ tool: "tasks_agent",
+ input: { message: "other" },
+ persistent: true,
+ }),
+ ).toEqual({
+ kind: "error",
+ error: 'Persistent tool "tasks_agent" requires an unconditional stub.',
+ });
+ });
+
+ it.each([
+ [{ tool: "list", outcome: { response: null } }],
+ [{ id: "a", tool: "", outcome: { response: null } }],
+ [{ id: "a", tool: "list" }],
+ [{ id: "a", tool: "list", response: null }],
+ [{ id: "a", tool: "list", responses: [null] }],
+ [{ id: "a", tool: "list", outcome: { response: null }, response: null }],
+ [{ id: "a", tool: "list", outcome: null }],
+ [{ id: "a", tool: "list", outcome: {} }],
+ [{ id: "a", tool: "list", outcome: { response: undefined } }],
+ [{ id: "a", tool: "list", outcome: { response: null, extra: true } }],
+ [{ id: "a", tool: "list", outcome: { throw: { name: "Error", message: "Not yet" } } }],
+ [{ id: "a", tool: "list", outcome: { response: null, throw: { message: "Not yet" } } }],
+ [{ id: "a", tool: "list", outcomes: [] }],
+ [{ id: "a", tool: "list", outcomes: [{ response: null }, {}] }],
+ [{ id: "a", tool: "list", outcomes: [{ response: null }, { throw: { message: "Not yet" } }] }],
+ [{ id: "a", tool: "list", outcome: { response: null }, outcomes: [{ response: null }] }],
+ [{ id: "a", tool: "list", outcome: { response: null }, typo: true }],
+ [{ id: "a", tool: "list", match: [], outcome: { response: null } }],
+ [{ id: "a", tool: "list", match: { x: 1 }, outcome: { response: null } }],
+ [{ id: "a", tool: "list", outcome: { response: { value: Number.NaN } } }],
+ [
+ { id: "a", tool: "list", outcome: { response: null } },
+ { id: "a", tool: "list", outcome: { response: null } },
+ ],
+ [{ id: "a", tool: "list", match: { x: { type: "strng" } }, outcome: { response: null } }],
+ [{ id: "a", tool: "list", match: { x: { pattern: "[" } }, outcome: { response: null } }],
+ [{ id: "a", tool: "list", match: { x: { minimum: "1" } }, outcome: { response: null } }],
+ [
+ {
+ id: "a",
+ tool: "list",
+ match: { x: { $ref: "https://example.com/schema" } },
+ outcome: { response: null },
+ },
+ ],
+ ])("rejects invalid rules and unsupported schemas before execution: %j", (...rules) => {
+ expect(() => parseToolStubs(rules)).toThrow();
+ });
+
+ it("normalizes absent options and preserves falsy JSON responses", () => {
+ expect(
+ parseToolStubs([
+ {
+ id: "constant",
+ tool: "list",
+ match: undefined,
+ outcome: { response: null },
+ outcomes: undefined,
+ },
+ {
+ id: "sequence",
+ tool: "list",
+ outcome: undefined,
+ outcomes: [{ response: false }, { response: 0 }, { response: "" }, { response: null }],
+ },
+ ]),
+ ).toEqual([
+ { id: "constant", tool: "list", outcome: { response: null } },
+ {
+ id: "sequence",
+ tool: "list",
+ outcomes: [{ response: false }, { response: 0 }, { response: "" }, { response: null }],
+ },
+ ]);
+ });
+
+ it("returns the response payload on every call, including keys named response or throw", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ {
+ id: "data",
+ tool: "lookup",
+ outcome: { response: { response: false, throw: "ordinary data" } },
+ },
+ ]),
+ );
+ for (const callId of ["first", "second"]) {
+ expect(playback.call({ callId, tool: "lookup", input: {} })).toEqual({
+ kind: "stub",
+ ruleId: "data",
+ position: 0,
+ outcome: { response: { response: false, throw: "ordinary data" } },
+ });
+ }
+ });
+
+ it.each([
+ [{ minimum: "1" }, "#/minimum", "number"],
+ [{ properties: { count: { minimun: 1 } } }, "#/properties/count/minimun", "Unsupported"],
+ [{ pattern: "[" }, "#/pattern", "regex"],
+ ])("identifies the rule, property, and invalid schema keyword: %j", (schema, path, detail) => {
+ expect(() =>
+ parseToolStubs([
+ { id: "valid", tool: "lookup", outcome: { response: null } },
+ {
+ id: "broken-filter",
+ tool: "lookup",
+ match: { filter: schema },
+ outcome: { response: null },
+ },
+ ]),
+ ).toThrow(
+ new RegExp(`Invalid matcher "filter" in tool stub "broken-filter" at ${path}: .*${detail}`),
+ );
+ });
+});
+
+it("does not consume another rule's sequence or advance on unmatched calls", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ {
+ id: "open",
+ tool: "lookup",
+ match: { status: { const: "open" } },
+ outcomes: [{ response: "open-1" }, { response: "open-2" }],
+ },
+ {
+ id: "closed",
+ tool: "lookup",
+ match: { status: { const: "closed" } },
+ outcomes: [{ response: "closed-1" }, { response: "closed-2" }],
+ },
+ ]),
+ );
+ const call = (callId: string, status: string) =>
+ playback.call({ callId, tool: "lookup", input: { status } });
+ expect(call("a", "open")).toMatchObject({ outcome: { response: "open-1" } });
+ expect(call("b", "absent")).toEqual({ kind: "real" });
+ expect(call("c", "closed")).toMatchObject({ outcome: { response: "closed-1" } });
+ expect(call("d", "open")).toMatchObject({ outcome: { response: "open-2" } });
+ expect(call("e", "closed")).toMatchObject({ outcome: { response: "closed-2" } });
+});
+
+it("requires the field even for a true constraint and lets an earlier broad match win", () => {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ { id: "present", tool: "lookup", match: { status: true }, outcome: { response: "any" } },
+ {
+ id: "open",
+ tool: "lookup",
+ match: { status: { const: "open" } },
+ outcome: { response: "open" },
+ },
+ ]),
+ );
+ expect(playback.call({ callId: "missing", tool: "lookup", input: {} })).toEqual({ kind: "real" });
+ expect(playback.call({ callId: "null", tool: "lookup", input: { status: null } })).toMatchObject({
+ outcome: { response: "any" },
+ });
+ expect(
+ playback.call({ callId: "overlap", tool: "lookup", input: { status: "open" } }),
+ ).toMatchObject({ kind: "stub", ruleId: "present", outcome: { response: "any" } });
+});
+
+it("includes the final visited string in the configuration size limit", () => {
+ expect(() =>
+ parseToolStubs([{ id: "x".repeat(1_000_001), tool: "list", outcome: { response: null } }]),
+ ).toThrow(/size/);
+});
diff --git a/packages/eve/src/tool-stubs/rules.ts b/packages/eve/src/tool-stubs/rules.ts
new file mode 100644
index 000000000..6ba74c379
--- /dev/null
+++ b/packages/eve/src/tool-stubs/rules.ts
@@ -0,0 +1,69 @@
+import { z } from "#compiled/zod/index.js";
+import { parseJsonValue } from "#shared/json.js";
+import { createStubValidator, validateStubConstraint } from "#tool-stubs/schema.js";
+import type { ToolStub } from "#tool-stubs/types.js";
+
+const ruleSchema = z.strictObject({
+ id: z.string().min(1),
+ tool: z.string().min(1),
+ match: z.record(z.string(), z.union([z.boolean(), z.record(z.string(), z.json())])).optional(),
+});
+
+const outcomeSchema = z.strictObject({ response: z.json() });
+
+const toolStubsSchema = z
+ .array(
+ z.union([
+ ruleSchema.extend({ outcome: outcomeSchema }),
+ ruleSchema.extend({ outcomes: z.tuple([outcomeSchema]).rest(outcomeSchema) }),
+ ]),
+ )
+ .max(100);
+
+/** Validates stubs before creating the session. */
+export function parseToolStubs(value: unknown): readonly ToolStub[] {
+ boundStubConfiguration(value);
+ const rules = toolStubsSchema.parse(parseJsonValue(value));
+ const ids = new Set<string>();
+ for (const rule of rules) {
+ if (ids.has(rule.id)) throw new Error(`Duplicate tool stub id "${rule.id}".`);
+ ids.add(rule.id);
+ if (["connection_execute", "connection_search"].includes(rule.tool.split("/").at(-1)!)) {
+ throw new Error(
+ "Cannot stub connection_execute or connection_search. Stub a specific connection operation, such as linear__list_issues.",
+ );
+ }
+ for (const [property, schema] of Object.entries(rule.match ?? {})) {
+ validateStubConstraint(schema, rule.id, property);
+ try {
+ createStubValidator(schema);
+ } catch (cause) {
+ throw new Error(`Could not create matcher "${property}" in tool stub "${rule.id}".`, {
+ cause,
+ });
+ }
+ }
+ }
+ return rules;
+}
+
+function boundStubConfiguration(value: unknown): void {
+ const pending = [{ value, depth: 0 }];
+ let nodes = 0;
+ let size = 0;
+ while (pending.length > 0) {
+ const entry = pending.pop()!;
+ if (++nodes > 20_000 || entry.depth > 32) {
+ throw new Error("Tool stubs exceed the size or nesting limit.");
+ }
+ if (typeof entry.value === "string") size += entry.value.length;
+ if (entry.value !== null && typeof entry.value === "object") {
+ for (const [key, child] of Object.entries(entry.value)) {
+ if (key === "__proto__") throw new Error("Tool stubs cannot contain prototype keys.");
+ size += key.length;
+ pending.push({ value: child, depth: entry.depth + 1 });
+ }
+ }
+ if (size > 1_000_000) throw new Error("Tool stubs exceed the size or nesting limit.");
+ }
+}
diff --git a/packages/eve/src/tool-stubs/schema.test.ts b/packages/eve/src/tool-stubs/schema.test.ts
new file mode 100644
index 000000000..dcb815e1a
--- /dev/null
+++ b/packages/eve/src/tool-stubs/schema.test.ts
@@ -0,0 +1,130 @@
+import { describe, expect, it } from "vitest";
+import type { JsonObject, JsonValue } from "#shared/json.js";
+import { parseToolStubs } from "#tool-stubs/rules.js";
+import { StubPlayback } from "#tool-stubs/playback.js";
+
+function matches(schema: JsonObject | boolean, value: JsonValue): boolean {
+ const playback = new StubPlayback(
+ parseToolStubs([
+ { id: "rule", tool: "lookup", match: { value: schema }, outcome: { response: "matched" } },
+ ]),
+ );
+ return playback.call({ callId: "call", tool: "lookup", input: { value } }).kind === "stub";
+}
+
+describe("supported tool stub constraints", () => {
+ it("matches model-written text with string patterns", () => {
+ const constraint = { type: "string", pattern: "\\b[Mm]ilk\\b" };
+ for (const title of ["Buy milk", "Buy milk at the store", "Buy whole-fat milk"]) {
+ expect(matches(constraint, title)).toBe(true);
+ }
+ expect(matches(constraint, "Walk the dog")).toBe(false);
+ expect(matches(constraint, "Buy milkshake")).toBe(false);
+ expect(matches(constraint, 42)).toBe(false);
+ });
+
+ it("matches scalar values without coercion", () => {
+ expect(matches({ enum: ["open", 1, false, null] }, "open")).toBe(true);
+ expect(matches({ enum: ["open", 1, false, null] }, "1")).toBe(false);
+ expect(matches({ type: "number", minimum: 1, maximum: 3 }, 2)).toBe(true);
+ expect(matches({ type: "number", minimum: 1, maximum: 3 }, 4)).toBe(false);
+ });
+
+ it("combines nested properties, array constraints, and alternatives", () => {
+ const constraint = {
+ type: "object",
+ required: ["tags"],
+ properties: {
+ tags: { type: "array", items: { type: "string" }, contains: { const: "urgent" } },
+ status: { anyOf: [{ const: "open" }, { const: "pending" }] },
+ },
+ };
+ expect(matches(constraint, { tags: ["urgent"], extra: true })).toBe(true);
+ expect(matches(constraint, { tags: ["urgent"], status: "pending" })).toBe(true);
+ expect(matches(constraint, { tags: [] })).toBe(false);
+ expect(matches(constraint, { tags: ["urgent"], status: "closed" })).toBe(false);
+ expect(matches(constraint, {})).toBe(false);
+ });
+
+ it("does not mutate schemas or arguments or apply defaults", () => {
+ const schema = { properties: { status: { default: "open" } } };
+ const value = { untouched: true };
+ expect(matches(schema, value)).toBe(true);
+ expect(value).toEqual({ untouched: true });
+ expect(schema).toEqual({ properties: { status: { default: "open" } } });
+ });
+});
+
+const invalid: JsonObject[] = [
+ { unknown: true },
+ { $ref: "#" },
+ { format: "email" },
+ { $schema: "https://json-schema.org/draft/2020-12/schema" },
+ { type: "strng" },
+ { type: [] },
+ { type: ["string", "string"] },
+ { type: ["string", "invalid"] },
+ { enum: [] },
+ {
+ enum: [
+ { a: 1, b: 2 },
+ { b: 2, a: 1 },
+ ],
+ },
+ { title: 1 },
+ { description: false },
+ { minimum: "1" },
+ { maximum: null },
+ { exclusiveMinimum: true },
+ { exclusiveMaximum: false },
+ { multipleOf: 0.1 },
+ { minLength: -1 },
+ { maxLength: 0.5 },
+ { pattern: 1 },
+ { pattern: "[" },
+ { minItems: -1 },
+ { maxItems: 0.5 },
+ { uniqueItems: true },
+ { items: [] },
+ { prefixItems: [] },
+ { prefixItems: [1] },
+ { contains: null },
+ { minContains: 1 },
+ { maxContains: 1 },
+ { minProperties: -1 },
+ { maxProperties: 0.5 },
+ { properties: [] },
+ { properties: { x: { unknown: true } } },
+ { patternProperties: { "^s_": true } },
+ { propertyNames: { pattern: "[" } },
+ { propertyNames: 1 },
+ { required: ["a", "a"] },
+ { required: [1] },
+ { additionalProperties: "false" },
+ { dependentSchemas: { card: { required: ["address"] } } },
+ { dependentRequired: { id: [] } },
+ { const: {} },
+ { const: [] },
+ { enum: ["open", {}] },
+ { enum: [[]] },
+ { required: ["constructor"] },
+ { properties: { toString: { type: "string" } } },
+ { allOf: [{ properties: { value: { const: [] } } }] },
+ { allOf: [] },
+ { anyOf: [1] },
+ { oneOf: [] },
+ { not: 1 },
+ { if: 1 },
+ // oxlint-disable-next-line unicorn/no-thenable
+ { then: 1 },
+ { else: 1 },
+ { allOf: [{ properties: { x: { pattern: "[" } } }] },
+];
+
+it.each(invalid)("rejects malformed or unsupported constraints at setup: %j", (schema) => {
+ expect(() =>
+ parseToolStubs([
+ { id: "bad", tool: "lookup", match: { value: schema }, outcome: { response: null } },
+ ]),
+ ).toThrow();
+});
diff --git a/packages/eve/src/tool-stubs/schema.ts b/packages/eve/src/tool-stubs/schema.ts
new file mode 100644
index 000000000..5ac22286d
--- /dev/null
+++ b/packages/eve/src/tool-stubs/schema.ts
@@ -0,0 +1,86 @@
+import { Validator } from "#compiled/@cfworker/json-schema/index.js";
+import type { JsonObject } from "#shared/json.js";
+
+const schema = { $ref: "#" };
+const schemaList = { type: "array", minItems: 1, items: schema };
+const propertyName = {
+ type: "string",
+ not: { enum: Object.getOwnPropertyNames(Object.prototype) },
+};
+const schemaMap = { type: "object", propertyNames: propertyName, additionalProperties: schema };
+const strings = { type: "array", uniqueItems: true, items: propertyName };
+const nonnegativeInteger = { type: "integer", minimum: 0 };
+const jsonType = { enum: ["array", "boolean", "integer", "null", "number", "object", "string"] };
+
+// The library validates data, but does not validate the schema itself:
+// { minimun: 10 } is ignored and { minimum: "10" } is not rejected.
+// Validate our supported, reference-free subset before using it to match inputs.
+const constraintSchema = new Validator(
+ {
+ anyOf: [
+ { type: "boolean" },
+ {
+ type: "object",
+ additionalProperties: false,
+ properties: {
+ type: {
+ anyOf: [jsonType, { type: "array", minItems: 1, uniqueItems: true, items: jsonType }],
+ },
+ const: { type: ["string", "number", "boolean", "null"] },
+ enum: {
+ type: "array",
+ minItems: 1,
+ uniqueItems: true,
+ items: { type: ["string", "number", "boolean", "null"] },
+ },
+ title: { type: "string" },
+ description: { type: "string" },
+ default: true,
+ minimum: { type: "number" },
+ maximum: { type: "number" },
+ exclusiveMinimum: { type: "number" },
+ exclusiveMaximum: { type: "number" },
+ minLength: nonnegativeInteger,
+ maxLength: nonnegativeInteger,
+ pattern: { type: "string", format: "regex" },
+ minItems: nonnegativeInteger,
+ maxItems: nonnegativeInteger,
+ items: schema,
+ prefixItems: schemaList,
+ contains: schema,
+ minProperties: nonnegativeInteger,
+ maxProperties: nonnegativeInteger,
+ properties: schemaMap,
+ propertyNames: schema,
+ required: strings,
+ additionalProperties: schema,
+ allOf: schemaList,
+ anyOf: schemaList,
+ oneOf: schemaList,
+ not: schema,
+ if: schema,
+ // JSON Schema uses `then` as a keyword.
+ // oxlint-disable-next-line unicorn/no-thenable
+ then: schema,
+ else: schema,
+ },
+ },
+ ],
+ },
+ "2020-12",
+);
+
+export function validateStubConstraint(value: unknown, ruleId: string, property: string): void {
+ const result = constraintSchema.validate(value);
+ if (!result.valid) {
+ const error = result.errors.at(-1)!;
+ const detail = error.keyword === "false" ? "Unsupported JSON Schema keyword." : error.error;
+ throw new Error(
+ `Invalid matcher "${property}" in tool stub "${ruleId}" at ${error.instanceLocation}: ${detail}`,
+ );
+ }
+}
+
+export function createStubValidator(value: JsonObject | boolean): Validator {
+ return new Validator(structuredClone(value), "2020-12");
+}
diff --git a/packages/eve/src/tool-stubs/types.ts b/packages/eve/src/tool-stubs/types.ts
new file mode 100644
index 000000000..3275fa7f8
--- /dev/null
+++ b/packages/eve/src/tool-stubs/types.ts
@@ -0,0 +1,51 @@
+import type { JsonObject, JsonValue } from "#shared/json.js";
+
+/** The JSON response returned for a matching tool call. */
+export interface ToolStubOutcome {
+ readonly response: JsonValue;
+}
+
+/** Stub outcomes selected by tool name and input fields. */
+export type ToolStub = {
+ readonly id: string;
+ /** Use list_tasks for a root tool or researcher/list_tasks for a local subagent's tool. */
+ readonly tool: string;
+ /** Each named input property must exist and satisfy its JSON Schema. */
+ readonly match?: Readonly<Record<string, JsonObject | boolean>>;
+} & (
+ | { readonly outcome: ToolStubOutcome; readonly outcomes?: never }
+ | {
+ readonly outcome?: never;
+ readonly outcomes: readonly [ToolStubOutcome, ...ToolStubOutcome[]];
+ }
+);
+
+export interface StubCall {
+ /** Combines the session, turn, and tool call IDs. */
+ readonly callId: string;
+ readonly tool: string;
+ readonly input: unknown;
+ readonly persistent?: boolean;
+}
+
+export type StubResult =
+ | { readonly kind: "real" }
+ | { readonly kind: "error"; readonly error: string }
+ | {
+ readonly kind: "stub";
+ readonly ruleId: string;
+ readonly position: number;
+ readonly outcome: ToolStubOutcome;
+ };
+
+/** Rules and workflow routing set by the server after it authorizes tool stubs. */
+export interface StubScope {
+ readonly token: string;
+ readonly rules: readonly ToolStub[];
+ readonly rootSessionId?: string;
+ readonly agentPath?: string;
+}
+
+export const STUB_CONTEXT_KEY = "eve.toolStubs";
+export const STUB_FAILURE_NAMESPACE = "eve.tool-stubs.failure";
+export const stubResponseNamespace = (callId: string): string => `eve.tool-stubs.${callId}`;