Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

7.7 KB
Raw Blame
import {
  latestTaskResult,
  outputOf,
  playScript,
  taskIdFromReceipt,
  type ScriptedCall,
} from "@eve-e2e/config/mock-script";
import type { MockModelRequest, MockModelResponse } from "eve/evals";
import { defineTool } from "eve/tools";
import { never } from "eve/tools/approval";
import { z } from "zod";

import {
  CORRECTED_MEASUREMENT,
  NOTEBOOK_CORRECTION,
  NOTEBOOK_DIRECTIVES,
  NOTEBOOK_ENTRY,
  NOTEBOOK_NAME,
  type NotebookDirective,
} from "../../constants.js";

// The notebook continuation script. Over three parent turns, the parent asks a
// keeper agent to remember a name, gives the same task a review that it
// cancels, then asks the same task for the name again. Only a keeper whose
// conversation survived the cancel can answer.
//
// The correction script is one parent turn: the parent asks a keeper to
// measure a pier, then corrects the pier by `taskId` while the keeper's
// measurement is still running, and waits for the corrected answer. The
// measurement outlasts the parent's next step, so the correction reaches the
// keeper's first turn while it works and that turn reads it.

const REMEMBER_ENTRY = `${NOTEBOOK_ENTRY} Alice named the tide station notebook ${NOTEBOOK_NAME}. Remember that name.`;
const REVIEW_ENTRY = `${NOTEBOOK_ENTRY} Please review the notebook pages before Alice's next question.`;
const RECALL_ENTRY = `${NOTEBOOK_ENTRY} What name did Alice give the tide station notebook?`;
const MEASURE_ENTRY = `${NOTEBOOK_ENTRY} Please measure the tide depth at the north pier for Alice.`;

/** Whether a message is one of the parent's notebook directives, such as `NOTEBOOK-RECALL notebook-keeper`. */
export function isNotebookDirective(message: string): boolean {
  return NOTEBOOK_DIRECTIVES.some((directive) => message.startsWith(`${directive} `));
}

/** Whether a message is one the parent sent a keeper; eve wraps the first one in a preamble. */
export function isNotebookEntry(message: string): boolean {
  return message.includes(`${NOTEBOOK_ENTRY} `);
}

/** The parent: plays the calls for the latest directive against the keeper tool it names. */
export function respondAsNotebookParent(request: MockModelRequest): MockModelResponse | string {
  const latest = [...request.userMessages].reverse().find(isNotebookDirective);
  if (latest === undefined) throw new Error("The notebook parent received no directive.");
  const [directive, tool] = latest.split(/\s+/u) as [NotebookDirective, string];
  const { calls, finish } = parentTurn(directive, tool);
  return playScript(request, calls, () => `NOTEBOOK-REPLY ${finish(request)}`);
}

function parentTurn(
  directive: NotebookDirective,
  tool: string,
): {
  readonly calls: readonly ScriptedCall[];
  readonly finish: (request: MockModelRequest) => string | undefined;
} {
  const keeperTask = (request: MockModelRequest) => taskIdFromReceipt(request, "notebook-remember");
  switch (directive) {
    case "NOTEBOOK-REMEMBER":
      return {
        calls: [
          { id: "notebook-remember", input: () => ({ message: REMEMBER_ENTRY }), name: tool },
          { id: "notebook-remember-wait", name: "eve__task_wait" },
        ],
        finish: (request) => latestTaskResult(request, tool),
      };
    case "NOTEBOOK-REVIEW":
      return {
        calls: [
          {
            id: "notebook-review",
            input: (request) => ({ message: REVIEW_ENTRY, taskId: keeperTask(request) }),
            name: tool,
          },
          {
            id: "notebook-review-wait",
            input: () => ({ timeoutSeconds: 2 }),
            name: "eve__task_wait",
          },
          {
            id: "notebook-cancel",
            input: (request) => ({ taskId: keeperTask(request) }),
            name: "eve__task_cancel",
          },
        ],
        finish: (request) => outputOf(request, "notebook-cancel"),
      };
    case "NOTEBOOK-RECALL":
      return {
        calls: [
          {
            id: "notebook-recall",
            input: (request) => ({ message: RECALL_ENTRY, taskId: keeperTask(request) }),
            name: tool,
          },
          { id: "notebook-recall-wait", name: "eve__task_wait" },
        ],
        finish: (request) => latestTaskResult(request, tool),
      };
    case "NOTEBOOK-CORRECT":
      return {
        calls: [
          { id: "notebook-measure", input: () => ({ message: MEASURE_ENTRY }), name: tool },
          {
            id: "notebook-correction",
            input: (request) => ({
              message: NOTEBOOK_CORRECTION,
              taskId: taskIdFromReceipt(request, "notebook-measure"),
            }),
            name: tool,
          },
          { id: "notebook-correction-wait", name: "eve__task_wait" },
        ],
        finish: (request) => latestTaskResult(request, tool),
      };
  }
}

/**
 * A keeper, local or remote: saves the name, reviews until its turn is
 * cancelled, and answers the name from its own conversation.
 */
export function respondAsNotebookKeeper(request: MockModelRequest): MockModelResponse | string {
  const message = request.lastUserMessage ?? "";
  if (message.includes(REMEMBER_ENTRY)) return "NOTEBOOK-SAVED";
  if (message.includes(REVIEW_ENTRY)) {
    const reviewId = `notebook-review-${String(request.userMessageCount)}`;
    return playScript(
      request,
      [{ id: reviewId, name: "notebook-review" }],
      () => "NOTEBOOK-REVIEWED",
    );
  }
  if (message.includes(RECALL_ENTRY)) {
    const remembers = request.userMessages.some((entry) => entry.includes(REMEMBER_ENTRY));
    return `NOTEBOOK-NAME=${remembers ? NOTEBOOK_NAME : "unknown"}`;
  }
  if (message.includes(MEASURE_ENTRY)) {
    const measureId = `notebook-measure-${String(request.userMessageCount)}`;
    return playScript(request, [{ id: measureId, name: "notebook-measure" }], measuredPier);
  }
  if (message.includes(NOTEBOOK_CORRECTION)) return CORRECTED_MEASUREMENT;
  throw new Error(`The notebook keeper received an unexpected message: ${message}`);
}

/** Reports the pier of the latest instruction the keeper has read. */
function measuredPier(request: MockModelRequest): string {
  const corrected = request.userMessages.some((entry) => entry.includes(NOTEBOOK_CORRECTION));
  return corrected ? CORRECTED_MEASUREMENT : "NOTEBOOK-DEPTH north pier 3.1 m";
}

const REVIEW_FALLBACK_MS = 60_000;

/** The keeper's review: works until its turn is cancelled, far longer than the parent waits. */
export function notebookReviewTool() {
  return defineTool({
    description: `Test fixture: reviews notebook pages until the review is cancelled. Only call when a ${NOTEBOOK_ENTRY} message asks for a notebook review.`,
    inputSchema: z.object({}),
    approval: never(),
    execute: (_input, ctx) => reviewUntilCancelled(ctx.abortSignal),
  });
}

function reviewUntilCancelled(signal: AbortSignal): Promise<string> {
  return new Promise((resolve, reject) => {
    const finish = setTimeout(() => resolve("The review finished."), REVIEW_FALLBACK_MS);
    const cancel = (): void => {
      clearTimeout(finish);
      reject(signal.reason);
    };
    if (signal.aborted) cancel();
    else signal.addEventListener("abort", cancel, { once: true });
  });
}

/** Long enough that the parent's correction arrives while the measurement runs. */
const MEASURE_MS = 5_000;

/** The keeper's measurement: finishes on its own after a few seconds. */
export function notebookMeasureTool() {
  return defineTool({
    description: `Test fixture: measures the tide depth at a pier. Only call when a ${NOTEBOOK_ENTRY} message asks for a measurement.`,
    inputSchema: z.object({}),
    approval: never(),
    execute: () => measureAfterDelay(),
  });
}

function measureAfterDelay(): Promise<string> {
  return new Promise((resolve) => {
    setTimeout(() => resolve("The measurement finished."), MEASURE_MS);
  });
}