The Open Framework for Building Agents
// Helpers for scripted mock responders: a scenario is a fixed sequence of tool
// calls with stable ids, played one model step at a time.
import type { EveEvalContext, MockModelRequest, MockModelResponse } from "eve/evals";
import { CALL_TOOL, SKILL_TOOL } from "./catalog-tools";
/** One tool call of a scripted scenario. Its fixed id marks it as done once its result is in the prompt. */
export interface ScriptedCall {
readonly id: string;
readonly name: string;
readonly input?: (request: MockModelRequest) => unknown;
}
/**
* Plays a scenario one model step at a time: the first call without a result
* runs next, and `finish` answers once every call has one.
*/
export function playScript(
request: MockModelRequest,
calls: readonly ScriptedCall[],
finish: (request: MockModelRequest) => MockModelResponse | string,
): MockModelResponse | string {
const done = new Set(request.toolResults.map((result) => result.id));
const next = calls.find((call) => !done.has(call.id));
if (next === undefined) return finish(request);
return { toolCalls: [{ id: next.id, input: next.input?.(request) ?? {}, name: next.name }] };
}
/** A scripted `eve__tool` call: a catalog tool by name, with its input. */
export function callTool(id: string, name: string, input?: object): ScriptedCall {
return { id, input: () => (input === undefined ? { name } : { input, name }), name: CALL_TOOL };
}
/** A scripted `eve__skill` call that loads the named skill. */
export function loadSkill(id: string, name: string): ScriptedCall {
return { id, input: () => ({ name }), name: SKILL_TOOL };
}
/** Loads each named skill with `eve__skill({ name })`, then replies with each skill's last line. */
export function loadSkills(
request: MockModelRequest,
names: readonly string[],
): MockModelResponse | string {
return playScript(
request,
names.map((skill) => loadSkill(`load-${skill}`, skill)),
(finished) =>
names.map((skill) => outputOf(finished, `load-${skill}`).trim().split("\n").at(-1)).join(" "),
);
}
/** Skips an eval whose scripted calls only the deterministic mock model makes. */
export function requireMockModel(
t: EveEvalContext,
reason = "Requires the deterministic mock model to issue the exact calls.",
): void {
if (process.env.EVE_E2E_MODEL !== "mock") t.skip(reason);
}
/** The output of the call with `id`, as the model sees it. */
export function outputOf(request: MockModelRequest, id: string): string {
const result = request.toolResults.find((entry) => entry.id === id);
if (result === undefined) throw new Error(`The script expected a result for call "${id}".`);
return typeof result.output === "string" ? result.output : JSON.stringify(result.output);
}
/** The task a receipt names, such as `Started task revise_plan-7k2m9q.` */
export function taskIdFromReceipt(request: MockModelRequest, callId: string): string {
const taskId = /Started task (\S+?)\.(?:\s|$)/u.exec(outputOf(request, callId))?.[1];
if (taskId === undefined) throw new Error(`Call "${callId}" returned no task receipt.`);
return taskId;
}
/** The body of the latest `<task_result>` block for `tool`, once a result has arrived. */
export function latestTaskResult(request: MockModelRequest, tool: string): string | undefined {
const pattern = new RegExp(
`<task_result [^>]*tool="${tool}"[^>]*>([\\s\\S]*?)</task_result>`,
"g",
);
for (const message of [...request.messages].reverse()) {
if (message.role !== "user") continue;
const bodies = [...message.text.matchAll(pattern)].map((match) => match[1]!);
if (bodies.length > 0) return bodies.at(-1);
}
return undefined;
}