Skip to content

The open-source app everyone uses to manage agents at work

7.5 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from "vitest";
import { redactTransferEvidence } from "./public-mcp-transfer-evidence.js";
import { assertSecretFree } from "./redaction.js";
import { hasReadLaterDocument, hasRequestedTaskEditSequence, runExpandedMcpScenario } from "./public-mcp-expanded-flow.js";
import { origin } from "./public-mcp-client.js";
import { isReadOnlyMcpCall } from "./public-mcp-grading.js";
import type { RunnerApi } from "./api.js";

const cases = ["task-edit", "documents", "files", "agent-config", "projects", "skills", "api", "permissions"];
afterEach(() => vi.unstubAllGlobals());

describe("expanded MCP independent evidence calibration", () => {
  it.each(cases)("accepts durable %s and rejects a plausible wrong outcome", async name => {
    for (const valid of [true, false]) {
      const checks: boolean[] = [];
      const nonce = "calibration";
      const bytes = Buffer.concat([Buffer.from([0, 255, 1, 128]), Buffer.from(`BINARY${nonce}`)]);
      const companyId = "company", taskId = "task";
      let activityReads = 0;
      const api = {
        post: async () => ({ id: "agent", name: "Settings agent" }),
        get: async (path: string) => {
          if (path.endsWith("/activity?limit=100")) return ++activityReads > 1 && valid ? taskHistory(nonce) : [];
          if (path.endsWith("/documents/report/revisions")) return valid ? [{ id: "old" }, { id: "new" }] : [];
          if (path.endsWith("/documents/report")) return { body: valid ? `GARDEN DOCUMENT${nonce}` : "GARDEN" };
          if (path.endsWith("/attachments")) return valid ? [{ originalFilename: "demo.mp4", byteSize: bytes.length }] : [];
          if (path.includes("instructions-bundle/file")) return { content: valid ? `Original operating instructions. INSTRUCTIONS${nonce}` : "Original operating instructions." };
          if (path.endsWith("/agents/agent")) return { title: name === "permissions" ? (valid ? "Original" : `Forbidden ${nonce}`) : (valid ? `Editor ${nonce}` : "Original"), budgetMonthlyCents: valid ? 1200 : 0 };
          if (path.endsWith("/projects")) return valid ? [{ name: `Project ${nonce}`, description: `Updated project ${nonce}` }] : [];
          if (path.includes("/files?path=SKILL.md")) return { content: valid ? `SKILL${nonce}` : "unchanged" };
          if (path.endsWith("/skills")) return valid ? [{ id: "skill", name: `Eval skill ${nonce}`, tagline: `Citations ${nonce}`, sharingScope: "company" }] : [];
          return { description: valid ? (name === "api" ? `API${nonce}` : `Edited ${nonce}`) : "Old", priority: valid ? "high" : "low", status: "done" };
        },
      } as unknown as RunnerApi;
      vi.stubGlobal("fetch", vi.fn(async (_url, init) => init?.method === "PUT"
        ? Response.json({ attachment: { id: "attachment" } })
        : new Response(valid ? bytes : Buffer.from("corrupted"))));
      await runExpandedMcpScenario({ id: `expanded-${name}`, api, companyId, taskId, title: "Task", marker: "GARDEN", nonce, secrets: [],
        check: (_id, passed) => checks.push(Boolean(passed)),
        converse: async (prompt, host) => {
          if (host) {
            for (const direction of ["upload", "download"]) await host.call("host_transfer_file", { direction, url: `${origin}/mcp/files/${direction}?ticket=fixture` });
          }
          // A plausible assistant success claim alone cannot make the fixture pass.
          return { prompt, final: `Done GARDEN DOCUMENT${nonce}; configuration permission required`, calls: name === "documents" ? (valid ? [{ name: "paperclip_read_document", arguments: { companyId, taskId, key: "report" }, result: { structuredContent: { document: { key: "report", body: `GARDEN DOCUMENT${nonce}` } } } }] : []) : ["paperclip_search_api", "paperclip_call_api"].map(name => ({ name, arguments: {}, result: {} })) };
        },
      });
      expect(checks.length).toBeGreaterThan(0);
      expect(checks.every(Boolean)).toBe(valid);
    }
  });

  it("rejects all mutation categories in read-only grading, including generic calls", () => {
    for (const name of ["paperclip_update_task", "paperclip_finish_task", "paperclip_block_task", "paperclip_update_agent", "paperclip_write_document", "paperclip_get_upload_url", "paperclip_write_skill_file", "paperclip_future_tool"]) {
      expect(isReadOnlyMcpCall({ name, arguments: {} })).toBe(false);
      expect(isReadOnlyMcpCall({ name: "paperclip_call_api", arguments: { operationId: name } })).toBe(false);
    }
    expect(isReadOnlyMcpCall({ name: "paperclip_call_api", arguments: { operationId: "paperclip_read_task" } })).toBe(true);
  });
});

 it("removes authorized transfer credentials without masking unrelated credential leaks", () => {
   const ticket = "temporary-transfer-ticket";
   const url = `https://paperclip.example/mcp/files/upload?ticket=${ticket}`;
   const value = { result: { url, content: [{ text: JSON.stringify({ url }) }] }, arguments: { url } };
   const safe = JSON.stringify(redactTransferEvidence(value, [ticket, url]));
   expect(safe).not.toContain(ticket);
   expect(safe).not.toContain(url);
   expect(() => assertSecretFree(safe, [ticket, url], "transfer")).not.toThrow();
   expect(() => assertSecretFree(JSON.stringify(redactTransferEvidence({ token: "provider-secret" }, [ticket, url])), ["provider-secret"], "unexpected")).toThrow("Secret leak");
   expect(value.result.url).toBe(url);
 });

function taskHistory(nonce: string) {
  return [
    { description: `Edited ${nonce}`, priority: "high" },
    { status: "blocked", unblockDescriptor: { owner: "board", action: `Supply source ${nonce}` } },
    { status: "done" },
  ].map((details, index) => ({ id: `event-${index}`, actorType: "user", action: "issue.updated", createdAt: new Date(1000 * (index + 1)).toISOString(), details }));
}
it("rejects wrong blocker ownership, action, actor and out-of-order completion", () => {
  expect(hasRequestedTaskEditSequence(taskHistory("n"), "n")).toBe(true);
  for (const mutate of [
    (h: any[]) => { h[1].details.unblockDescriptor.owner = "agent"; },
    (h: any[]) => { h[1].details.unblockDescriptor.action = "Wrong action"; },
    (h: any[]) => { h[1].actorType = "agent"; },
    (h: any[]) => { h[2].createdAt = new Date(500).toISOString(); },
    (h: any[]) => { h.unshift({ ...h[2], id: "earlier-finish", createdAt: new Date(500).toISOString() }); },
  ]) {
    const history = taskHistory("n"); mutate(history);
    expect(hasRequestedTaskEditSequence(history, "n")).toBe(false);
  }
});

it("requires a real later report read and rejects writes, errors, wrong targets and missing references", () => {
  const read = { name: "paperclip_read_document", arguments: { companyId: "c", taskId: "t", key: "report" },
    result: { structuredContent: { document: { key: "report", body: "GARDEN DOCUMENT" } } } };
  const grade = (calls: any[]) => hasReadLaterDocument({ prompt: "Read", final: "GARDEN DOCUMENT", calls }, "c", "t", "GARDEN", "DOCUMENT");
  expect(grade([read])).toBe(true);
  expect(grade([{ name: "paperclip_call_api", arguments: { companyId: "c", operationId: read.name, arguments: read.arguments }, result: read.result }])).toBe(true);
  for (const calls of [[], [{ ...read, arguments: { ...read.arguments, companyId: "foreign" } }],
    [{ ...read, arguments: { ...read.arguments, taskId: "other" } }], [{ ...read, result: { ...read.result, isError: true } }],
    [{ ...read, result: { structuredContent: { document: { key: "report", body: "GARDEN" } } } }],
    [read, { name: "paperclip_write_document", arguments: {}, result: {} }],
    [read, { name: "paperclip_call_api", arguments: { operationId: "paperclip_update_task" }, result: {} }],
  ]) expect(grade(calls)).toBe(false);
});