Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

3.9 KB
Raw Blame
import { defineEval, type EveEvalTargetHandle } from "eve/evals";
import { satisfies } from "eve/evals/expect";

interface CreateSessionResponse {
  readonly ok: true;
  readonly sessionId: string;
  readonly status: "accepted";
}

const PRINCIPAL_A = "Bearer e2e-create-once-a";
const PRINCIPAL_B = "Bearer e2e-create-once-b";

export default defineEval({
  description:
    "Concurrent create candidates converge on one owner, serial retries resolve it, and issuers remain isolated.",
  async test(t) {
    const operationId = `session-create-idempotency-${crypto.randomUUID()}`;
    const message = `CREATE-ONCE-INITIAL-${crypto.randomUUID()}`;
    const [first, concurrent] = await Promise.all([
      createSession(t.target, PRINCIPAL_A, operationId, message),
      createSession(t.target, PRINCIPAL_A, operationId, message),
    ]);

    const acceptedCandidates = new Set([first.sessionId, concurrent.sessionId]);
    let replay: CreateSessionResponse | undefined;
    for (const delayMs of [100, 200, 400, 800, 1_600]) {
      await t.sleep(delayMs);
      const retry = await createSession(t.target, PRINCIPAL_A, operationId, message);
      if (acceptedCandidates.has(retry.sessionId)) {
        replay = retry;
        break;
      }
      acceptedCandidates.add(retry.sessionId);
    }
    if (replay === undefined) {
      throw new Error("The operation owner was not resolvable after startup.");
    }

    const otherIssuer = await createSession(t.target, PRINCIPAL_B, operationId, message);
    await t.require(
      otherIssuer,
      satisfies(
        (value: CreateSessionResponse) => value.sessionId !== replay.sessionId,
        "the same operation under another issuer owns a distinct session",
      ),
    );

    const [firstTurn, issuerTurn] = await Promise.all([
      t.target.watchTurn(replay.sessionId).result(),
      t.target.watchTurn(otherIssuer.sessionId).result(),
    ]);
    // A real model may take extra tool steps on the opaque marker message, so
    // duplicate execution is detected by a repeated first step, not a step count.
    firstTurn.expectOk();
    firstTurn.event("message.received", { count: 1, data: { message } });
    firstTurn.event("step.started", { count: 1, data: { stepIndex: 0 } });
    issuerTurn.expectOk();
    issuerTurn.event("message.received", { count: 1, data: { message } });
    issuerTurn.event("step.started", { count: 1, data: { stepIndex: 0 } });

    const probe = `CREATE-ONCE-PROBE-${crypto.randomUUID()}`;
    const liveProbe = t.target.watchTurn(firstTurn.sessionId, {
      startIndex: firstTurn.events.length,
    });
    await continueSession(t.target, PRINCIPAL_A, firstTurn.sessionId, probe);
    const probeTurn = await liveProbe.result();
    probeTurn.expectOk();
    probeTurn.event("message.received", { count: 1, data: { message: probe } });
    probeTurn.event("step.started", { count: 1, data: { stepIndex: 0 } });
  },
});

async function createSession(
  target: EveEvalTargetHandle,
  authorization: string,
  operationId: string,
  message: string,
): Promise<CreateSessionResponse> {
  const response = await target.fetch("/eve/v1/session", {
    body: JSON.stringify({ message, operationId }),
    headers: { authorization, "content-type": "application/json" },
    method: "POST",
  });
  const text = await response.text();
  if (!response.ok) throw new Error(`POST /eve/v1/session failed (${response.status}): ${text}`);
  return JSON.parse(text) as CreateSessionResponse;
}

async function continueSession(
  target: EveEvalTargetHandle,
  authorization: string,
  sessionId: string,
  message: string,
): Promise<void> {
  const response = await target.fetch(`/eve/v1/session/${encodeURIComponent(sessionId)}`, {
    body: JSON.stringify({ message }),
    headers: { authorization, "content-type": "application/json" },
    method: "POST",
  });
  if (!response.ok) {
    throw new Error(`POST continuation failed (${response.status}): ${await response.text()}`);
  }
}