Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

14.9 KB
Raw Blame
/**
 * The Agent API's management surface: the API tab's routes
 * (`/api/projects/:projectId/agents/:agentId/api`), the Agent list's mark, the deletion
 * cascades, and the admin's switch.
 *
 * Scenarios:
 * - Given a Project member, the settings read back (off, keyed, allow-all, no keys for an Agent
 *   never configured); changing them — the switch, keyless access, the approval mode, a key —
 *   is the owner's alone, and a PUT with one invalid field changes nothing.
 * - Given a created key, its secret is in the create response only: the listing shows its
 *   prefix, name, creator and when it was last used — stamped by a run that authenticated
 *   with it — and never the secret; a name outside 1-64 characters is refused, and deleting a
 *   key the Agent does not have is 404.
 * - Given the Agent list, only the Agent whose switch is on carries the API mark, and a newly
 *   created Agent carries none.
 * - Given an exposed Agent deleted and created again under the same id, it starts off with no
 *   keys, and the old key opens nothing.
 * - Given a Project with keys whose owner is deleted, the deletion goes through.
 * - Given the admin, the Agent API switch reads on by default and round-trips through
 *   GET/PUT /api/admin/settings; a non-admin cannot change it.
 * - Given the admin's switch off, a Project owner who cannot read the admin settings learns it
 *   from the API tab's settings, on a read and on a write alike.
 * - Given the local API token, the credential every tool subprocess holds: it reads an Agent's
 *   settings and keys, but turning the API on, creating a key, deleting one and Try it are each
 *   403 `human_required` and change nothing, run nothing and create no Session; the owner's
 *   sign-in then does each of them.
 * - Given the local API token on the admin settings: it changes every setting but
 *   `agentApiEnabled`; a body carrying that is 403 `human_required` alone, and none of its
 *   other fields is written.
 */
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type {
  AgentApiKeyCreateResponse,
  AgentApiResponse,
  AgentsResponse,
  ServerSettingsResponse,
} from "../src/api/types.js";
import { apiClient, createTestApp, loginAdmin, provisionUser, tokenClient } from "./helpers.js";
import type { TestApp } from "./helpers.js";
import { adoptSession, fakeSession, uniqueSessionId } from "./fixtures/session.js";

type Client = ReturnType<typeof apiClient>;

describe("Agent API settings", () => {
  let t: TestApp;
  let admin: Client;
  let owner: Client;
  let member: Client;
  let projectId: string;
  const tab = (agentId: string) => `/api/projects/${projectId}/agents/${agentId}/api`;
  const read = async (client: Client, agentId: string) =>
    ((await (await client.get(tab(agentId))).json()) as AgentApiResponse).api;

  beforeAll(async () => {
    t = await createTestApp();
    admin = apiClient(t.app, (await loginAdmin(t.app)).cookie);
    owner = apiClient(t.app, (await provisionUser(t.app, "api_owner")).cookie);
    member = apiClient(t.app, (await provisionUser(t.app, "api_member")).cookie);
    projectId = "api_owner-default_project";
    expect(
      (await owner.post(`/api/projects/${projectId}/members`, { userId: "api_member" })).status,
    ).toBe(201);
    for (const agentId of ["keyed_agent", "quiet_agent"]) {
      expect((await owner.post(`/api/projects/${projectId}/agents`, { agentId })).status).toBe(201);
    }
  });
  afterAll(async () => {
    await t.deps.manager.shutdown();
    await t.cleanup();
  });

  it("a member reads the settings, only the owner changes them (enabled, open, approval mode)", async () => {
    expect(await read(member, "default_agent")).toEqual({
      enabled: false,
      open: false,
      approvalMode: "allow-all",
      keys: [],
    });
    for (const res of [
      await member.put(tab("default_agent"), { enabled: true }),
      await member.post(`${tab("default_agent")}/keys`, { name: "mine" }),
    ]) {
      expect(res.status).toBe(403);
    }
    const put = await owner.put(tab("default_agent"), {
      enabled: true,
      open: true,
      approvalMode: "read-only",
    });
    expect(put.status).toBe(200);
    expect(((await put.json()) as AgentApiResponse).api).toMatchObject({
      enabled: true,
      open: true,
      approvalMode: "read-only",
    });
    // One invalid field and nothing is written.
    const bad = await owner.put(tab("default_agent"), {
      enabled: false,
      approvalMode: "sometimes",
    });
    expect(bad.status).toBe(400);
    expect(await read(member, "default_agent")).toMatchObject({
      enabled: true,
      open: true,
      approvalMode: "read-only",
    });
    // Omitted fields keep their value.
    await owner.put(tab("default_agent"), { open: false });
    expect(await read(owner, "default_agent")).toMatchObject({ enabled: true, open: false });
    // Someone outside the Project learns nothing.
    const stranger = apiClient(t.app, (await provisionUser(t.app, "api_stranger")).cookie);
    expect((await stranger.get(tab("default_agent"))).status).toBe(404);
    // An Agent that does not exist has no settings to read.
    expect((await owner.get(tab("ghost_agent"))).status).toBe(404);
  });

  it("creating a key returns the secret once; the listing shows the prefix and never the secret", async () => {
    await owner.put(tab("keyed_agent"), { enabled: true });
    const res = await owner.post(`${tab("keyed_agent")}/keys`, { name: "ci runner" });
    expect(res.status).toBe(201);
    const { key, secret } = (await res.json()) as AgentApiKeyCreateResponse;
    expect(secret).toMatch(/^penguin_[A-Za-z0-9_-]{43}$/);
    expect(key).toMatchObject({
      name: "ci runner",
      prefix: secret.slice(0, 16),
      createdBy: "api_owner",
      lastUsedAt: null,
    });
    const listing = await member.get(tab("keyed_agent"));
    const text = await listing.text();
    expect(text).not.toContain(secret);
    expect((JSON.parse(text) as AgentApiResponse).api.keys).toEqual([key]);

    // A run that authenticated with the key stamps it.
    const sessionId = uniqueSessionId();
    adoptSession(t.deps, fakeSession(sessionId), {
      projectId,
      agentId: "keyed_agent",
      client: "api",
    });
    const run = await t.app.request(`/api/amsp/v1/agents/${projectId}/keyed_agent/runs`, {
      method: "POST",
      headers: { authorization: `Bearer ${secret}`, "content-type": "application/json" },
      body: JSON.stringify({ session_id: sessionId, input: "hi" }),
    });
    expect(run.status).toBe(200);
    await run.text();
    const [used] = (await read(owner, "keyed_agent")).keys;
    expect(Number.isNaN(Date.parse(used!.lastUsedAt ?? ""))).toBe(false);

    for (const name of ["", "   ", "x".repeat(65)]) {
      expect((await owner.post(`${tab("keyed_agent")}/keys`, { name })).status).toBe(400);
    }
    const missing = await owner.delete(`${tab("keyed_agent")}/keys/not-a-key`);
    expect(missing.status).toBe(404);
    expect(((await missing.json()) as { error: { code: string } }).error.code).toBe(
      "key_not_found",
    );
    expect((await read(owner, "keyed_agent")).keys).toHaveLength(1);
  });

  it("the Agent list marks an enabled Agent and nothing else", async () => {
    await owner.put(tab("quiet_agent"), { enabled: true });
    await owner.put(tab("quiet_agent"), { enabled: false });
    const created = await owner.post(`/api/projects/${projectId}/agents`, { agentId: "fresh_one" });
    expect(((await created.json()) as { agent: { apiEnabled: boolean } }).agent.apiEnabled).toBe(
      false,
    );
    const { agents } = (await (
      await member.get(`/api/projects/${projectId}/agents`)
    ).json()) as AgentsResponse;
    const marked = Object.fromEntries(agents.map((a) => [a.agentId, a.apiEnabled]));
    expect(marked).toMatchObject({
      default_agent: true,
      keyed_agent: true,
      quiet_agent: false,
      fresh_one: false,
    });
  });

  it("deleting the Agent removes its switch and keys", async () => {
    await owner.post(`/api/projects/${projectId}/agents`, { agentId: "short_lived" });
    await owner.put(tab("short_lived"), { enabled: true, open: true, approvalMode: "deny-all" });
    const { secret } = (await (
      await owner.post(`${tab("short_lived")}/keys`, { name: "k" })
    ).json()) as AgentApiKeyCreateResponse;
    expect((await owner.delete(`/api/projects/${projectId}/agents/short_lived`)).status).toBe(202);

    await owner.post(`/api/projects/${projectId}/agents`, { agentId: "short_lived" });
    expect(await read(owner, "short_lived")).toEqual({
      enabled: false,
      open: false,
      approvalMode: "allow-all",
      keys: [],
    });
    await owner.put(tab("short_lived"), { enabled: true });
    const old = await t.app.request(`/api/amsp/v1/agents/${projectId}/short_lived`, {
      headers: { authorization: `Bearer ${secret}` },
    });
    expect(old.status).toBe(401);
  });

  it("a Project's switches and keys go with it, so its owner's account can be deleted", async () => {
    const doomed = apiClient(t.app, (await provisionUser(t.app, "api_doomed")).cookie);
    const base = `/api/projects/api_doomed-default_project/agents/default_agent/api`;
    expect((await doomed.put(base, { enabled: true })).status).toBe(200);
    expect((await doomed.post(`${base}/keys`, { name: "k" })).status).toBe(201);
    expect((await admin.delete("/api/admin/users/api_doomed")).status).toBe(204);
    expect(
      t.deps.db
        .prepare("SELECT COUNT(*) AS n FROM agent_api_keys WHERE project_id = ?")
        .get("api_doomed-default_project"),
    ).toEqual({ n: 0 });
  });

  it("the API tab tells an owner who is no admin that the admin has the API off", async () => {
    expect((await owner.get("/api/admin/settings")).status).toBe(403);
    expect(
      ((await (await owner.get(tab("default_agent"))).json()) as AgentApiResponse).serverEnabled,
    ).toBe(true);
    await admin.put("/api/admin/settings", { agentApiEnabled: false });
    try {
      const read = (await (await member.get(tab("default_agent"))).json()) as AgentApiResponse;
      expect(read.serverEnabled).toBe(false);
      // The Agent's own switch is kept, and a write answers the same way.
      expect(read.api.enabled).toBe(true);
      const write = await owner.put(tab("default_agent"), { open: false });
      expect(((await write.json()) as AgentApiResponse).serverEnabled).toBe(false);
    } finally {
      await admin.put("/api/admin/settings", { agentApiEnabled: true });
    }
  });

  /** The answer is a refusal for want of a person's sign-in. */
  const humanRequired = async (res: Response) => {
    expect(res.status).toBe(403);
    expect(((await res.json()) as { error: { code: string } }).error.code).toBe("human_required");
  };

  it("the local API token reads an Agent's API but cannot change it or try it; the owner's sign-in can", async () => {
    // The admin's own Project: the token authenticates as the admin, who owns it.
    const mine = "/api/projects/default_project/agents/default_agent/api";
    const token = tokenClient(t.app, t.deps.authService.localApiToken()!);
    const readByToken = async () => {
      const res = await token.get(mine);
      expect(res.status).toBe(200);
      return ((await res.json()) as AgentApiResponse).api;
    };

    await humanRequired(await token.put(mine, { enabled: true, open: true }));
    expect(await readByToken()).toMatchObject({ enabled: false, open: false });
    expect((await admin.put(mine, { enabled: true })).status).toBe(200);
    expect(await readByToken()).toMatchObject({ enabled: true, open: false });

    await humanRequired(await token.post(`${mine}/keys`, { name: "carried out" }));
    expect((await readByToken()).keys).toEqual([]);
    const created = await admin.post(`${mine}/keys`, { name: "ci" });
    expect(created.status).toBe(201);
    const { key } = (await created.json()) as AgentApiKeyCreateResponse;
    expect((await readByToken()).keys).toEqual([key]);

    await humanRequired(await token.delete(`${mine}/keys/${key.keyId}`));
    expect((await readByToken()).keys).toEqual([key]);
    expect((await admin.delete(`${mine}/keys/${key.keyId}`)).status).toBe(204);
    expect((await readByToken()).keys).toEqual([]);

    // Try it: refused before anything runs or is created, on a new conversation and an old one.
    let runs = 0;
    const sessionId = uniqueSessionId();
    adoptSession(
      t.deps,
      fakeSession(sessionId, {
        async *run() {
          runs += 1;
        },
      }),
      { projectId: "default_project", agentId: "default_agent", client: "api" },
    );
    const sessions = t.deps.sessionsRepo.listByAgent("default_project", "default_agent").length;
    await humanRequired(await token.post(`${mine}/try`, { input: "hi" }));
    await humanRequired(await token.post(`${mine}/try`, { session_id: sessionId, input: "hi" }));
    expect(runs).toBe(0);
    expect(t.deps.sessionsRepo.listByAgent("default_project", "default_agent")).toHaveLength(
      sessions,
    );
    const tried = await admin.post(`${mine}/try`, { session_id: sessionId, input: "hi" });
    expect(tried.status).toBe(200);
    await tried.text();
    expect(runs).toBe(1);
  });

  it("the local API token changes every admin setting but agentApiEnabled, which is refused with the rest of its body", async () => {
    const token = tokenClient(t.app, t.deps.authService.localApiToken()!);
    const read = async () =>
      ((await (await token.get("/api/admin/settings")).json()) as ServerSettingsResponse).settings;
    const before = await read();
    const flipped = {
      proxyForApp: !before.proxyForApp,
      proxyForAgent: !before.proxyForAgent,
    };
    try {
      expect((await token.put("/api/admin/settings", flipped)).status).toBe(200);
      expect(await read()).toEqual({ ...before, ...flipped });

      await humanRequired(
        await token.put("/api/admin/settings", {
          agentApiEnabled: !before.agentApiEnabled,
          proxyForAgent: before.proxyForAgent,
          proxyForApp: before.proxyForApp,
        }),
      );
      expect(await read()).toEqual({ ...before, ...flipped });
    } finally {
      await token.put("/api/admin/settings", {
        proxyForAgent: before.proxyForAgent,
        proxyForApp: before.proxyForApp,
      });
    }
  });

  it("the admin switch round-trips through GET/PUT /api/admin/settings", async () => {
    const get = async () =>
      ((await (await admin.get("/api/admin/settings")).json()) as ServerSettingsResponse).settings;
    expect((await get()).agentApiEnabled).toBe(true);
    const off = await admin.put("/api/admin/settings", { agentApiEnabled: false });
    expect(((await off.json()) as ServerSettingsResponse).settings.agentApiEnabled).toBe(false);
    expect((await get()).agentApiEnabled).toBe(false);
    expect((await admin.put("/api/admin/settings", { agentApiEnabled: "no" })).status).toBe(400);
    expect((await owner.put("/api/admin/settings", { agentApiEnabled: true })).status).toBe(403);
    expect((await get()).agentApiEnabled).toBe(false);
    await admin.put("/api/admin/settings", { agentApiEnabled: true });
    expect((await get()).agentApiEnabled).toBe(true);
  });
});