Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
9.2 KB
Raw Blame
import { expect } from "vitest";
import { spec } from "@omnirush/testkit";
import { sessionErrorCard, sessionSubmitErrorIsolation } from "../worlds/chat.ts";

const test = spec.world(sessionErrorCard);
const submitTest = spec.world(sessionSubmitErrorIsolation, { timeout: 600_000 });
const STORAGE_TITLE = "Storage error reported";
const STORAGE_DESCRIPTION = "A storage limit was reported by the task runtime or a connected service. This does not necessarily mean your computer is full. Check the affected service or workspace before freeing local disk space.";

// Values from the seeded payload (eval.session_error.seed): an Anthropic 429
// with a JSON response body. None of these appear in the plain card text.
const CARD_TEXT = "Rate limit reached for claude-sonnet-4-5";
const DIAGNOSTIC_LINES = ["Error type: APIError", "Status: 429", "Provider: anthropic", "Code: rate_limit_error", "Retries: 3"];
const REQUEST_ID = "req_01JZK4W9N7X2Q8M3V5T6B1C0DE";
// Rendered with CSS uppercase, so compare case-insensitively.
const DEBUG_PANEL_TITLE = /react session debug/i;

const detailsToggle = { testId: "session-error-details-toggle" };
const detailsPanel = { testId: "session-error-details" };
const statusLine = { text: /Status: 429/ };
const requestId = { text: new RegExp(REQUEST_ID) };

test("session error cards expose provider diagnostics only in Developer mode", async ({ user, probe, step, world, place }) => {
  // Toggle Developer mode the way a person does: command palette → "Enable/Disable Developer Mode".
  const toggleDeveloperMode = async (next: "on" | "off") => {
    const label = next === "on" ? /enable developer mode/i : /disable developer mode/i;
    await user.press(place.kind !== "daytona" && process.platform === "darwin" ? "Meta+K" : "Control+K");
    await user.click({ role: "option", label });
    await probe.eventually(() => probe.storage("omnirush.developerMode"), {
      until: (value) => String(value) === (next === "on" ? "1" : "0"),
      within: 10_000,
      label: `Developer mode persisted ${next}`,
    });
  };

  await step("with Developer mode off, a failed turn shows only the plain error card", async () => {
    await user.see({ text: new RegExp(CARD_TEXT) });
    expect(String(await probe.storage("omnirush.developerMode"))).not.toBe("1");
    await user.notSee(detailsToggle);
    await user.notSee(statusLine);
    await user.notSee(requestId);
    await user.notSee({ text: DEBUG_PANEL_TITLE });
  });

  await step("enabling Developer mode reaches the session surface without a reload", async () => {
    await toggleDeveloperMode("on");
    await user.see(detailsToggle);
    await user.see({ text: new RegExp(CARD_TEXT) });
    await user.see({ text: DEBUG_PANEL_TITLE });
    // The disclosure starts collapsed: the toggle is present, the panel is not.
    await user.notSee(detailsPanel);
    await user.notSee(statusLine);
  });

  await step("opening Technical details shows the full provider diagnostic payload with a copy action", async () => {
    await user.click(detailsToggle);
    await user.see(detailsPanel);
    for (const line of DIAGNOSTIC_LINES) await user.see({ text: new RegExp(line.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")) });
    // The request id sits inside the raw response-body block, so match it within that block's text.
    await user.see(requestId);
    await user.see({ role: "button", label: /copy details/i });
  });

  await step("disabling Developer mode removes diagnostics while the error card remains", async () => {
    await toggleDeveloperMode("off");
    await user.notSee(detailsToggle, { timeoutMs: 10_000 });
    await user.see({ text: new RegExp(CARD_TEXT) });
    await user.notSee(detailsPanel);
    await user.notSee(statusLine);
    await user.notSee(requestId);
    await user.notSee({ text: DEBUG_PANEL_TITLE });
  });

  const storageErrors: Array<"disk-full" | "database-error"> = ["disk-full", "database-error"];
  for (const kind of storageErrors) {
    await step(`${kind} shows recovery guidance and keeps the stack trace in Developer mode`, async () => {
      await world.seedStorageError(kind);
      const title = kind === "disk-full" ? STORAGE_TITLE : "OmniRush.ai couldn’t access its saved data";
      await user.see({ text: title });
      await user.see({ text: kind === "disk-full" ? STORAGE_DESCRIPTION : /check the available disk space/ });
      await user.notSee({ text: /effect\/sql\/SqlError/ });
      await user.notSee({ text: /at runLoop/ });
      await user.notSee(detailsToggle);
      await user.notSee({ text: "Not enough disk space" });
      if (kind === "database-error") await user.notSee({ text: STORAGE_TITLE });
      await toggleDeveloperMode("on");
      await user.click(detailsToggle);
      await user.see({ text: /effect\/sql\/SqlError/ });
      await user.see({ text: /at runLoop/ });
      await user.see({ role: "button", label: /copy details/i });
      await toggleDeveloperMode("off");
      await user.see({ text: title });
      await user.notSee(detailsPanel);
      await user.notSee({ text: /at runLoop/ });
    });
    await step(`${kind} banner hides the stack trace outside Developer mode`, async () => {
      await world.seedStorageError(kind, "banner");
      const title = kind === "disk-full" ? STORAGE_TITLE : "OmniRush.ai couldn’t access its saved data";
      await user.see({ testId: "session-error-card" });
      await user.see({ text: title });
      await user.notSee({ text: /at runLoop/ });
      await toggleDeveloperMode("on");
      await user.see({ text: /effect\/sql\/SqlError/ });
      await user.see({ text: /at runLoop/ });
      await toggleDeveloperMode("off");
      await user.see({ text: title });
      await user.notSee({ text: /at runLoop/ });
    });
  }

});

submitTest("a delayed submit error stays with its owner while another retained task remains sendable", async ({ user, probe, step, world }) => {
  const a = world.sessionA.sessionId;
  const b = world.sessionB.sessionId;
  const banner = { testId: "session-error-card" };
  const select = async (title: string, sessionId: string) => {
    await user.click({ text: title });
    await probe.eventually(() => world.selectedSurface(), {
      within: 15_000, label: "the selected task owns the visible surface", until: value => value.sessionId === sessionId,
    });
  };
  const readyToSend = (sessionId: string) => probe.eventually(() => world.selectedSurface(), {
    within: 15_000, label: "the selected task can send", until: value => value.sessionId === sessionId && value.runEnabled,
  });
  const releaseFailure = async (count: number) => {
    await world.failHeldSubmissions();
    await probe.eventually(() => world.readSubmissions(), {
      within: 15_000, label: "the held SDK error response finishes", until: value => value.finished === count,
    });
    await world.settleResponse();
  };

  await step("A's real submit stays pending while B is selected and editable", async () => {
    await user.type("composer", "Save the first task output.", { verify: true });
    await readyToSend(a);
    await user.click({ role: "button", label: "Run task" });
    await probe.eventually(() => world.readSubmissions(), {
      within: 30_000, label: "A's actual submit is intercepted before its response", until: value => value.held === 1,
    });
    await select(world.sessionB.title, b);
    await user.type("composer", world.promptB, { verify: true });
    await readyToSend(b);
    expect(world.readSubmissions().finished).toBe(0);
  });

  await step("A's late upstream storage failure neither shows a B banner nor blocks B's one send", async () => {
    await releaseFailure(1);
    await user.notSee(banner);
    await user.notSee({ text: STORAGE_TITLE });
    await user.see("composer", { editable: true, text: world.promptB });
    // Do not poll past an incorrect disabled state after the response settled.
    expect(await world.selectedSurface()).toEqual({ sessionId: b, runEnabled: true });
    await user.click({ role: "button", label: "Run task" });
    await user.see({ text: world.replyB }, { timeoutMs: 60_000 });
    expect(world.readSubmissions().requests.filter(request => request.sessionId === b)).toHaveLength(1);
    expect(world.readSubmissions().requests.find(request => request.sessionId === b)?.body).toContain(world.promptB);
    await user.notSee(banner);
  });

  await step("a failure delivered while A owns the surface remains visible with neutral storage guidance", async () => {
    await select(world.sessionA.title, a);
    await user.type("composer", "Save the first task output again.", { replace: true, verify: true });
    await readyToSend(a);
    await user.click({ role: "button", label: "Run task" });
    await probe.eventually(() => world.readSubmissions(), {
      within: 30_000, label: "A's own second submit reaches the same HTTP boundary", until: value => value.held === 2,
    });
    await releaseFailure(2);
    await user.see(banner);
    await user.see({ text: STORAGE_TITLE });
    await user.see({ text: STORAGE_DESCRIPTION });
    await user.notSee({ text: /EDQUOT/ });
    expect(await world.selectedSurface()).toEqual({ sessionId: a, runEnabled: false });
    await select(world.sessionB.title, b);
    await user.see({ text: world.replyB });
    await user.notSee(banner);
    expect(world.readSubmissions().requests.filter(request => request.sessionId === b)).toHaveLength(1);
  });
});