Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

14.9 KB
Raw Blame
import { afterEach, describe, expect, test } from "bun:test";
import {
  type CapabilityRow,
  DEFAULT_CAPABILITIES,
  resolveCapability,
} from "../src/admin/capabilities";
import { decideFromSnapshot } from "../src/admin/controls";
import { DEFAULT_ENTERPRISE_SETTINGS } from "../src/admin/settings-store";
import { decideSignInMethod } from "../src/auth";
import {
  evaluateActionPolicy,
  type PolicyContext,
  setPolicyOverlay,
} from "../src/computer/policy";
import {
  effectiveNetworkPolicy,
  pushEgressPolicies,
} from "../src/computer/policy-network";
import type { ComputerProvider } from "../src/computer/provider";
import { scrubCommand } from "../src/telemetry/scrub";

const row = (
  scopeKind: CapabilityRow["scopeKind"],
  scopeId: string,
  capability: string,
  allowed: boolean,
): CapabilityRow => ({ scopeKind, scopeId, capability, allowed });

describe("capability resolution", () => {
  const user = { role: "user" as const, groups: [] };

  test("no rows means the built-in default, which keeps today's behaviour", () => {
    expect(resolveCapability([], user, "cloudBrowser")).toMatchObject({
      allowed: true,
      decidedBy: "default",
    });
    expect(resolveCapability([], user, "teamBots").allowed).toBe(
      DEFAULT_CAPABILITIES.teamBots,
    );
  });

  test("organization is the baseline, a role replaces it", () => {
    const rows = [
      row("organization", "", "localComputer", false),
      row("role", "admin", "localComputer", true),
    ];
    expect(resolveCapability(rows, user, "localComputer").allowed).toBe(false);
    expect(
      resolveCapability(rows, { role: "admin", groups: [] }, "localComputer"),
    ).toMatchObject({ allowed: true, decidedBy: "role" });
  });

  test("a group only widens", () => {
    const rows = [
      row("organization", "", "slackTeams", false),
      row("group", "sales", "slackTeams", true),
      row("organization", "", "cloudBrowser", true),
      row("group", "contractors", "cloudBrowser", false),
    ];
    expect(
      resolveCapability(
        rows,
        { role: "user", groups: ["sales"] },
        "slackTeams",
      ),
    ).toMatchObject({ allowed: true, decidedBy: "group", scopeId: "sales" });
    expect(resolveCapability(rows, user, "slackTeams").allowed).toBe(false);
    // A group row of false narrows nothing.
    expect(
      resolveCapability(
        rows,
        { role: "user", groups: ["contractors"] },
        "cloudBrowser",
      ).allowed,
    ).toBe(true);
  });
});

function snapshot(
  overrides: Partial<Parameters<typeof decideFromSnapshot>[0] & object> = {},
) {
  return {
    rows: [],
    admins: new Set<string>(),
    groupsByUser: new Map<string, string[]>(),
    settings: DEFAULT_ENTERPRISE_SETTINGS,
    network: [],
    loadedAt: new Date(),
    ...overrides,
  };
}

function context(overrides: Partial<PolicyContext> = {}): PolicyContext {
  return {
    tool: { name: "computer_navigate" },
    bot: { id: "bot" },
    actor: { id: "person" },
    page: { url: "https://example.com/", host: "example.com" },
    intent: "navigate",
    key: "",
    element: { ref: "", role: "", name: "", type: "" },
    file: { path: "", name: "", extension: "" },
    command: "",
    mcp: { server: "", tool: "", effect: "" },
    initiator: { kind: "person", id: "" },
    ...overrides,
  };
}

describe("the computer boundary overlay", () => {
  afterEach(() => setPolicyOverlay(null));

  test("a snapshot that never loaded refuses (fail closed)", () => {
    expect(decideFromSnapshot(undefined, context())?.forward).toBe(false);
  });

  test("nothing configured says nothing, so the boundary's own rules decide", () => {
    expect(decideFromSnapshot(snapshot(), context())).toBeNull();
  });

  test("browser, network, computer and Bots switches each refuse their actions", () => {
    const off = (capability: string) =>
      snapshot({ rows: [row("organization", "", capability, false)] });
    expect(decideFromSnapshot(off("cloudBrowser"), context())?.matched).toBe(
      "capability:cloudBrowser",
    );
    expect(decideFromSnapshot(off("cloudNetwork"), context())?.matched).toBe(
      "capability:cloudNetwork",
    );
    const shell = context({
      tool: { name: "computer_run_command" },
      intent: "run_command",
      command: "ls",
    });
    expect(decideFromSnapshot(off("cloudComputer"), shell)?.matched).toBe(
      "capability:cloudComputer",
    );
    expect(decideFromSnapshot(off("cloudBrowser"), shell)).toBeNull();
    expect(decideFromSnapshot(off("useBots"), shell)?.matched).toBe(
      "capability:useBots",
    );
  });

  test("the computer switch refuses downloading a workspace file", () => {
    const off = (capability: string) =>
      snapshot({ rows: [row("organization", "", capability, false)] });
    const download = context({
      tool: { name: "computer_download_file" },
      intent: "download_file",
    });
    expect(decideFromSnapshot(off("cloudComputer"), download)?.matched).toBe(
      "capability:cloudComputer",
    );
    expect(decideFromSnapshot(off("cloudBrowser"), download)).toBeNull();
  });

  test("a group grant reaches the snapshot's members", () => {
    const current = snapshot({
      rows: [
        row("organization", "", "cloudBrowser", false),
        row("group", "eng", "cloudBrowser", true),
      ],
      groupsByUser: new Map([["person", ["eng"]]]),
    });
    expect(decideFromSnapshot(current, context())).toBeNull();
    expect(
      decideFromSnapshot(current, context({ actor: { id: "other" } }))?.forward,
    ).toBe(false);
  });

  test("the MCP allowlist refuses servers not on it", () => {
    const current = snapshot({
      settings: {
        ...DEFAULT_ENTERPRISE_SETTINGS,
        mcpAllowlist: { enabled: true, servers: ["jira"] },
      },
    });
    const call = (server: string) =>
      context({
        tool: { name: `${server}__search` },
        intent: "read_tool",
        page: { url: "", host: "" },
        mcp: { server, tool: "search", effect: "read" },
      });
    expect(decideFromSnapshot(current, call("jira"))).toBeNull();
    expect(decideFromSnapshot(current, call("shadow-mcp"))?.matched).toBe(
      "mcp_allowlist",
    );
  });

  test("a navigation off the allowlist is refused before it reaches the computer", () => {
    const current = snapshot({
      network: [
        {
          scopeKind: "organization",
          scopeId: "",
          mode: "allowlist_only",
          rules: [{ type: "domain", value: "example.com" }],
          locked: false,
          updatedBy: null,
          updatedAt: new Date().toISOString(),
        },
      ],
    });
    expect(decideFromSnapshot(current, context())).toBeNull();
    expect(
      decideFromSnapshot(
        current,
        context({ page: { url: "https://evil.test/", host: "evil.test" } }),
      )?.matched,
    ).toBe("network:organization");
  });

  test("installed, it is final even over a permissive dry-run boundary", () => {
    const current = snapshot({
      rows: [row("organization", "", "cloudBrowser", false)],
    });
    setPolicyOverlay((ctx) => decideFromSnapshot(current, ctx));
    const decision = evaluateActionPolicy(
      { mode: "dry-run", deny: [], allow: ["true"] },
      context(),
    );
    expect(decision).toMatchObject({ allowed: false, forward: false });
  });

  test("an overlay that throws refuses", () => {
    setPolicyOverlay(() => {
      throw new Error("boom");
    });
    expect(
      evaluateActionPolicy(
        { mode: "enforce", deny: [], allow: ["true"] },
        context(),
      ).forward,
    ).toBe(false);
  });
});

describe("network policy selection", () => {
  const organization = {
    scopeKind: "organization" as const,
    scopeId: "",
    mode: "allowlist_only" as const,
    rules: [{ type: "domain", value: "example.com" }],
    locked: false,
    updatedBy: null,
    updatedAt: "",
  };
  const group = {
    ...organization,
    scopeKind: "group" as const,
    scopeId: "research",
    mode: "allow_all" as const,
    rules: [],
  };

  test("a group policy replaces the organization's, unless it is locked", () => {
    expect(
      effectiveNetworkPolicy(
        [organization, group],
        { groups: ["research"] },
        true,
      ),
    ).toMatchObject({ mode: "allow_all", from: "group:research" });
    expect(
      effectiveNetworkPolicy(
        [{ ...organization, locked: true }, group],
        { groups: ["research"] },
        true,
      ),
    ).toMatchObject({ mode: "allowlist_only", from: "organization" });
  });

  test("no network capability is deny_all whatever the policy says", () => {
    expect(
      effectiveNetworkPolicy([group], { groups: ["research"] }, false).mode,
    ).toBe("deny_all");
  });

  test("the push goes to running computers only, never waking one", async () => {
    const sent: { url: string; body: string; bot: string | null }[] = [];
    const provider = {
      list: async () => [
        { botId: "a", status: "running", url: "http://10.0.0.1:4100" },
        { botId: "b", status: "stopped", url: "http://10.0.0.2:4100" },
      ],
    } as unknown as ComputerProvider;
    const report = await pushEgressPolicies({
      provider,
      token: "t",
      policyForBot: async () => ({ mode: "deny_all", rules: [] }),
      fetchImpl: (async (url: string, init: RequestInit) => {
        sent.push({
          url,
          body: String(init.body),
          bot: new Headers(init.headers).get("x-openbot-bot-id"),
        });
        return new Response("{}");
      }) as unknown as typeof fetch,
    });
    expect(report.pushed).toEqual(["a"]);
    expect(sent).toEqual([
      {
        url: "http://10.0.0.1:4100/egress-policy",
        body: JSON.stringify({ policy: { mode: "deny_all", rules: [] } }),
        bot: "a",
      },
    ]);
  });
});

describe("SSO required", () => {
  const decide = (method: string, email: string, required: boolean) =>
    decideSignInMethod({
      method,
      email,
      ssoRequired: async () => required,
      initialAdminEmails: ["[email protected]"],
    });

  test("off admits everybody, on admits only an enterprise provider", async () => {
    expect(await decide("oauth", "[email protected]", false)).toBe("allow");
    expect(await decide("oauth", "[email protected]", true)).toBe("refuse");
    expect(await decide("email-password", "[email protected]", true)).toBe("refuse");
    expect(await decide("sso-saml", "[email protected]", true)).toBe("allow");
    expect(await decide("sso-oidc", "[email protected]", true)).toBe("allow");
  });

  test("a configured administrator is let in as break-glass", async () => {
    expect(await decide("oauth", "[email protected]", true)).toBe("break_glass");
  });

  test("settings that cannot be read refuse", async () => {
    expect(
      await decideSignInMethod({
        method: "oauth",
        email: "[email protected]",
        ssoRequired: async () => {
          throw new Error("database down");
        },
        initialAdminEmails: [],
      }),
    ).toBe("refuse");
  });
});

describe("scrubbing commands", () => {
  const cases: [string, string][] = [
    [
      "curl -H 'Authorization: Bearer abc.def' https://api.example.com",
      "curl -H 'Authorization: Bearer [REDACTED]' https://api.example.com",
    ],
    [
      "export GITHUB_TOKEN=ghp_abcdefghijklmnopqrstuvwxyz0123456789",
      "export GITHUB_TOKEN=[REDACTED]",
    ],
    [
      "psql postgres://app:hunter2@db:5432/app",
      "psql postgres://app:[REDACTED]@db:5432/app",
    ],
    ["mysql -uroot -phunter2 app", "mysql -uroot -p[REDACTED] app"],
    // `-P` is mysql's port, not its password: the attached port must survive and the real
    // password later in the same command must still be redacted (gh-815).
    ["mysql -h db -P3306 -phunter2 app", "mysql -h db -P3306 -p[REDACTED] app"],
    ["mysql -uroot -p3306 app", "mysql -uroot -p[REDACTED] app"],
    ["mysqldump -phunter2 db", "mysqldump -p[REDACTED] db"],
    [
      "tool --password hunter2 --verbose",
      "tool --password [REDACTED] --verbose",
    ],
    [
      "AWS_SECRET_ACCESS_KEY=abc123 aws s3 ls",
      "AWS_SECRET_ACCESS_KEY=[REDACTED] aws s3 ls",
    ],
    [
      "curl -u admin:s3cret https://x.test",
      "curl -u admin:[REDACTED] https://x.test",
    ],
    ["echo sk-proj-abcdefghijklmnopqrstuvwxyz", "echo [REDACTED]"],
    [
      `curl -d '{"user":"ann","password":"hunter2"}' https://x.test`,
      `curl -d '{"user":"ann","password":[REDACTED]}' https://x.test`,
    ],
    [
      `curl -d '{"api_key": "abcd1234", "n": 1}' https://x.test`,
      `curl -d '{"api_key": [REDACTED], "n": 1}' https://x.test`,
    ],
    [
      'curl -d "{\\"client_secret\\":\\"hunter2\\"}" https://x.test',
      'curl -d "{\\"client_secret\\":[REDACTED]}" https://x.test',
    ],
    [
      "curl -uadmin:s3cret https://x.test",
      "curl -uadmin:[REDACTED] https://x.test",
    ],
    [
      "curl --user=admin:s3cret https://x.test",
      "curl --user=admin:[REDACTED] https://x.test",
    ],
    [
      'curl -u "admin:s3 cret" https://x.test',
      "curl -u admin:[REDACTED] https://x.test",
    ],
    [
      "curl --proxy-user bob:hunter2 -x http://proxy.test:3128 https://x.test",
      "curl --proxy-user bob:[REDACTED] -x http://proxy.test:3128 https://x.test",
    ],
    // `-p` bare asks mysql for the password and leaves the database name after it alone: the two are
    // separated by a space, and the attached form is what carries a password.
    ["mysql -h db -p", "mysql -h db -p"],
    ["mysql -u root -p mydb", "mysql -u root -p mydb"],
    ["docker login -u bob -p hunter2", "docker login -u bob -p [REDACTED]"],
    ["sshpass -p hunter2 ssh user@host", "sshpass -p [REDACTED] ssh user@host"],
    ["redis-cli -a hunter2 SET k v", "redis-cli -a [REDACTED] SET k v"],
    ["redis-cli -ahunter2 SET k v", "redis-cli -a[REDACTED] SET k v"],
    ["redis-cli --pass hunter2 SET k v", "redis-cli --pass [REDACTED] SET k v"],
    ["redis://:hunter2@cache:6379", "redis://:[REDACTED]@cache:6379"],
    // Each of those rules ends where its command does. A line break starts another command, whose
    // `-p` is a port again; a line continued with a backslash does not. sshpass's options end at
    // the command it wraps, and that command's own `-p` and `-P` are ports too.
    [
      "sshpass -p hunter2 ssh -p 2222 user@host",
      "sshpass -p [REDACTED] ssh -p 2222 user@host",
    ],
    [
      "sshpass -f pw.txt ssh -p 2222 user@host",
      "sshpass -f pw.txt ssh -p 2222 user@host",
    ],
    ["sshpass -e scp -P 2222 f host:", "sshpass -e scp -P 2222 f host:"],
    [
      "docker login -u bob r.test\ndocker run -p 8080:80 nginx",
      "docker login -u bob r.test\ndocker run -p 8080:80 nginx",
    ],
    [
      "docker login -u bob \\\n  -p hunter2 r.test",
      "docker login -u bob \\\n  -p [REDACTED] r.test",
    ],
    ["redis-cli PING\nls -a /tmp", "redis-cli PING\nls -a /tmp"],
  ];
  for (const [input, output] of cases) {
    test(input, () => expect(scrubCommand(input)).toBe(output));
  }
  test("an ordinary command is untouched", () => {
    expect(scrubCommand("ls -la /workspace && git status")).toBe(
      "ls -la /workspace && git status",
    );
  });
});