Skip to content

The open-source app everyone uses to manage agents at work

5.8 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from "vitest";
import { assistantUsage, recordAssistantUsage, runAssistant } from "./public-mcp-model.js";
import { aggregateCampaignBilling, summarizeExecutionBilling } from "./billing.js";
import type { RunnerE2EResult } from "./types.js";

afterEach(() => vi.unstubAllGlobals());
describe("external assistant evidence and billing", () => {
  it("rejects missing usage instead of silently treating paid work as free", () => {
    const usage = assistantUsage("openai", "gpt-5.4-mini");
    expect(() => recordAssistantUsage(usage, "gpt-5.4-mini", NaN, 20, 0)).toThrow("cost is unknown");
    expect(() => recordAssistantUsage(usage, "undefined", 200, 20, 0)).toThrow("cost is unknown");
  });
  it("prices uncached, cached and output tokens separately without calling estimates reported costs", () => {
    const usage = assistantUsage("openai", "gpt-5.4-nano");
    recordAssistantUsage(usage, "gpt-5.4-nano", 1000, 100, 2000);
    expect(usage.estimatedCostUsd).toBeCloseTo(0.000365);
    const result: RunnerE2EResult = {
      schema: "paperclip.runner-e2e.result/v2", executionId: "public-mcp.assistant-codex-mini.local.delegate-retrieve",
      attempt: 1, status: "passed", profileId: "assistant-codex-mini", caseId: "delegate-retrieve", provider: "codex", model: "gpt-5.4-mini", runtimeMode: "legacy",
      startedAt: "2026-09-30T00:00:00Z", finishedAt: "2026-09-30T00:00:01Z", cleanup: "passed",
      runIds: ["worker-run"], usage: { inputTokens: 100, outputTokens: 10, costUsd: 0.01 }, environmentId: "local", durationMs: 1000, publicMcp: usage,
    };
    const billing = summarizeExecutionBilling(result);
    expect(billing.reportedCostUsd).toBe(0.01);
    expect(billing.assistant?.requests).toBe(1);
    expect(billing.observedAndEstimatedCostUsd).toBeCloseTo(0.010365);
    const aggregate = aggregateCampaignBilling([result, result]);
    expect(aggregate.assistant?.requests).toBe(2);
    expect(aggregate.assistant?.estimatedCostUsd).toBeCloseTo(0.00073);
    expect(aggregate.reportedLlmCostUsd).toBe(0.02);
  });

  it("retains tool evidence and final text while omitting provider reasoning", async () => {
    const fetchMock = vi.fn()
      .mockResolvedValueOnce(Response.json({ model: "gpt-5.4-nano", usage: { input_tokens: 200, output_tokens: 20 }, output: [
        { type: "reasoning", summary: [{ text: "private fixture reasoning" }] },
        { type: "function_call", name: "paperclip_connection", call_id: "one", arguments: "{}" },
      ] }))
      .mockResolvedValueOnce(Response.json({ model: "gpt-5.4-nano", usage: { input_tokens: 250, output_tokens: 10 }, output: [{ type: "message", content: [{ type: "output_text", text: "Connected to Team." }] }] }));
    vi.stubGlobal("fetch", fetchMock);
    const observe = vi.fn();
    const call = vi.fn().mockResolvedValue({ companyId: "team" });
    const usage = assistantUsage("openai", "gpt-5.4-nano");
    const result = await runAssistant({ usage, credential: "unit-test-key", prompt: "Which team?", tools: [], call, deadlineAt: Date.now() + 10_000, observe });
    expect(result.final).toBe("Connected to Team.");
    expect(call).toHaveBeenCalledExactlyOnceWith("paperclip_connection", {});
    expect(JSON.stringify(observe.mock.calls)).not.toContain("private fixture reasoning");
    expect(JSON.stringify(observe.mock.calls)).not.toContain("unit-test-key");
    expect(usage.requests).toBe(2);
  });

  it("retains a failed partial turn and does not call a missing final answer a pass", async () => {
    vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ model: "gpt-5.4-nano", usage: { input_tokens: 200, output_tokens: 20 }, output: [] })));
    const observe = vi.fn();
    await expect(runAssistant({ usage: assistantUsage("openai", "gpt-5.4-nano"), credential: "unit-test-key", prompt: "Do work", tools: [], call: vi.fn(), deadlineAt: Date.now() + 10_000, observe })).rejects.toThrow("no visible final answer");
    expect(observe).toHaveBeenCalledTimes(2);
  });

  it("persists paid usage and completed tools before the next provider request", async () => {
    vi.stubGlobal("fetch", vi.fn()
      .mockResolvedValueOnce(Response.json({ model: "gpt-5.4-mini", usage: { input_tokens: 200, output_tokens: 20 }, output: [{ type: "function_call", name: "paperclip_connection", call_id: "one", arguments: "{}" }] }))
      .mockRejectedValueOnce(new Error("fixture connection interrupted")));
    const snapshots: Array<{ calls: number; requests: number }> = [];
    const usage = assistantUsage("openai", "gpt-5.4-mini");
    await expect(runAssistant({ usage, credential: "unit-test-key", prompt: "Which team?", tools: [],
      call: async () => ({ companyId: "company" }), deadlineAt: Date.now() + 10_000,
      observe: async turn => { snapshots.push({ calls: turn.calls.length, requests: usage.requests }); },
    })).rejects.toThrow("fixture connection interrupted");
    expect(snapshots).toEqual([{ calls: 0, requests: 1 }, { calls: 1, requests: 1 }, { calls: 1, requests: 1 }]);
    expect(usage.estimatedCostUsd).toBeGreaterThan(0);
  });
  it("shares the 16-request ceiling across fresh conversations in one cell", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => Response.json({ model: "gpt-5.4-mini", usage: { input_tokens: 10, output_tokens: 10 }, output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }] }));
    vi.stubGlobal("fetch", fetchMock);
    const usage = assistantUsage("openai", "gpt-5.4-mini");
    const input = { usage, credential: "unit-test-key", prompt: "Fresh conversation", tools: [], call: vi.fn(), deadlineAt: Date.now() + 10_000, observe: vi.fn() };
    for (let i = 0; i < 16; i++) await runAssistant(input);
    await expect(runAssistant(input)).rejects.toThrow("shared 16-request per-cell budget");
    expect(fetchMock).toHaveBeenCalledTimes(16);
    expect(usage.requests).toBe(16);
  });

});