Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
6.5 KB
Raw Blame
import { expect } from "vitest";
import { spec } from "@omnirush/testkit";
import { computerMentions } from "../worlds/chat.ts";

const test = spec.world(computerMentions);

test("computer mentions steer tasks through Connect and Automations names the computer", async ({ world, user, probe, step, evidence }) => {
  await step("the mention menu explains both computers without starting a task", async () => {
    await user.type("composer", "@");
    await user.see({ text: "Start a task on your cloud computer" });
    await user.see({ text: "Start a task on your connected desktop computer" });
    await user.screenshot();
    expect((await probe.composer()).userMessageCount).toBe(0);
    await user.type("composer", "cl", { replace: false });
    await user.see({ text: "Start a task on your cloud computer" });
    await user.notSee({ text: "Start a task on your connected desktop computer" });
    await user.click({ role: "button", label: /@cloud/ });
    await user.type("composer", "COMPUTER-CLOUD-TASK Summarize the project notes.");
    await user.press("Enter");
    await user.see({ text: "Received computer task.", nth: 0 }, { timeoutMs: 90_000 });
    await user.see({ text: /^@cloud COMPUTER-CLOUD-TASK Summarize the project notes\.$/ });
    await user.notSee({ text: /The user selected @cloud|Use OmniRush.ai Connect search_capabilities/ });
    await user.reload();
    await user.see({ text: /^@cloud COMPUTER-CLOUD-TASK Summarize the project notes\.$/ });
    await user.notSee({ text: /The user selected @cloud|Use OmniRush.ai Connect search_capabilities/ });
    evidence.recordAssertionEvidence("Cloud mention stays compact after reload", "The user sees the exact @cloud task text before and after reload; generated routing instructions are absent in both views.", true);
  });

  await step("typing desktop directly works without selecting the menu", async () => {
    await user.click({ role: "button", label: "New session" });
    await user.type("composer", "@desktop COMPUTER-DESKTOP-TASK Summarize my local project notes.");
    await user.press("Enter");
    await user.see({ text: "Received computer task.", nth: 0 }, { timeoutMs: 90_000 });
    await user.click({ role: "button", label: "New session" });
    await user.type("composer", "COMPUTER-PLAIN-TASK Explain the address person@cloud and the word desktop.");
    await user.press("Enter");
    await user.see({ text: "Received computer task.", nth: 0 }, { timeoutMs: 90_000 });
  });

  await step("skill mentions keep generated instructions out of the user message", async () => {
    for (const { token, visible, hidden } of [
      { token: "[skill summarize]", visible: "summarize COMPUTER-PLAIN-TASK Summarize notes.", hidden: /Load \[skill summarize\] and follow its instructions/ },
      { token: "[connect-skill summarize|Summarize|Team tools|skill:summarize]", visible: "/summarize COMPUTER-PLAIN-TASK Summarize notes.", hidden: /skill:summarize/ },
    ]) {
      await user.click({ role: "button", label: "New session" });
      await user.type("composer", `${token} COMPUTER-PLAIN-TASK Summarize notes.`);
      await user.press("Enter");
      await user.see({ text: "Received computer task.", nth: 0 }, { timeoutMs: 90_000 });
      await user.see({ text: visible });
      await user.notSee({ text: hidden });
      await user.reload();
      await user.see({ text: visible });
      await user.notSee({ text: hidden });
    }
    evidence.recordAssertionEvidence("Skill labels survive reload without instruction expansion", "Both local and Connect skill labels remain visible before and after reload, with generated instructions absent from chat.", true);
  });

  await step("computer handoffs reach Connect, while ordinary text makes no tool call", async () => {
    const messages = await world.submittedParts();
    expect(messages).toEqual([
      { visible: "@cloud COMPUTER-CLOUD-TASK Summarize the project notes.", routing: [expect.stringContaining('target "cloud"')] },
      { visible: "@desktop COMPUTER-DESKTOP-TASK Summarize my local project notes.", routing: [expect.stringContaining('target "desktop"')] },
      { visible: "COMPUTER-PLAIN-TASK Explain the address person@cloud and the word desktop.", routing: [] },
      { visible: "[skill summarize] COMPUTER-PLAIN-TASK Summarize notes.", routing: [expect.stringContaining("Load [skill summarize] and follow its instructions.")] },
      { visible: "/summarize COMPUTER-PLAIN-TASK Summarize notes.", routing: [expect.stringContaining("skill:summarize")] },
    ]);
    const calls = await probe.toolCalls(world.den.mocks.agent);
    expect(calls.map(({ name, args }) => ({ name, args }))).toEqual([
      { name: "search_capabilities", args: { query: "remote-session:create" } },
      { name: "execute_capability", args: { name: "remote-session:create", body: { target: "cloud", prompt: "COMPUTER-CLOUD-TASK Summarize the project notes." } } },
      { name: "search_capabilities", args: { query: "remote-session:create" } },
      { name: "execute_capability", args: { name: "remote-session:create", body: { target: "desktop", prompt: "COMPUTER-DESKTOP-TASK Summarize my local project notes." } } },
    ]);
    await user.notSee({ text: /Use OmniRush.ai Connect search_capabilities/ });
    evidence.recordAssertionEvidence("Computer mentions reach the Connect boundary", "Menu-selected @cloud and typed @desktop submit distinct synthetic Connect routing instructions. The Connect witness serves search and create calls with the matching target and full task. An email address makes no tool call, and routing instructions stay out of the visible chat.", true);
  });

  await step("Automations shows where the task runs and explains desktop availability", async () => {
    await user.click({ role: "button", label: /^Automations$/ });
    await user.see({ text: "Schedule tasks on your desktop or cloud computer. See where each task runs and how it went." });
    await user.see({ text: "Daily project summary" });
    await user.see({ text: /^Desktop computer$/ });
    await user.notSee({ text: /Scheduled durably|headlessly|fixed Desktop/ });
    await user.screenshot();
    await user.click({ role: "button", label: /Daily project summary/ });
    await user.see({ text: "Runs on your desktop computer. Keep OmniRush.ai open and connected at the scheduled time." });
    evidence.recordAssertionEvidence("Automation placement is visible and understandable", "The automation list labels its desktop computer; the detail explains that OmniRush.ai must stay open and connected without runtime terminology.", true);
  });
});