Multiplayer agent harness for work.
Document and test deployment broker credentials
Josh Francefb47dbafec3f
diff --git a/docs/deployment-broker-credentials.md b/docs/deployment-broker-credentials.mdnew file mode 100644index 000000000..410fa414f--- /dev/null+++ b/docs/deployment-broker-credentials.md@@ -0,0 +1,59 @@+# Deployment broker credentials++An operator can give every agent in a deployment access to shared API keys without storing them in the+settings database. Set them in the server environment, the same way as `MODEL_GATEWAY_API_KEY`. The core+reads them once at startup.++Agents use these credentials only through the credential broker. They never see the secret, and org admins+can see them in Settings but can't edit, delete or replace them.++## Format++`BROKER_CREDENTIALS` is a JSON array. Each entry names the env var that holds its secret:++```json+[+ {+ "slug": "web-search",+ "displayName": "Web search",+ "host": "api.search.example",+ "secretEnv": "WEB_SEARCH_API_KEY",+ "injection": { "header": "X-API-KEY", "scheme": "" },+ "allowedMethods": ["POST"],+ "allowedPathPrefixes": ["/search"],+ "maxRequestsPerDay": 500+ }+]+```++| Field | Required | Meaning |+| --------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------- |+| `slug` | yes | Lowercase kebab-case name agents use. Must be unique. |+| `host` | yes | Bare hostname the broker pins requests to (subdomains match, as for stored credentials). |+| `secretEnv` | yes | Name of the env var holding the secret. It must be set and non-empty. |+| `displayName` | no | Label shown in Settings and to agents. Defaults to the slug. |+| `injection` | no | `header` and `scheme` for the secret. Defaults to `Authorization: Bearer <secret>`; set `scheme` to `""` to send the raw secret. |+| `allowedMethods` | no | Allowed HTTP methods. Defaults to `GET`. |+| `allowedPathPrefixes` | no | Allowed path prefixes. Defaults to any path. |+| `maxRequestsPerDay` | no | Cap on calls let through per UTC day, shared by every core process. Calls over the cap get `429 daily_cap_reached`. |++Unknown fields, a missing secret, a duplicate slug or any invalid value stop the core at startup with an+error naming the entry. Errors never include secret values.++## Behaviour++- **Who can use them:** every conversation whose participants all belong to the org, as if the credential+ had an org-wide grant. Published apps can't use them.+- **Delivery:** broker only. They are never injected into sandbox environments.+- **Precedence:** a deployment slug is reserved. Lookups return the deployment credential even if the+ settings database still holds one with the same slug, and admins can't create, edit or delete a stored+ credential with that slug.+- **Not for decision providers:** a decision provider can't name a deployment credential.+- **Usage:** broker calls are logged and summarised in Settings like any other credential.++## Operating++1. Put each secret in your secret store and expose it to the core as its own env var.+2. Set `BROKER_CREDENTIALS` to the JSON array and restart the core.+3. To rotate a key, change the secret's env var and restart. To remove a credential, drop its entry and+ restart.diff --git a/test/deployment-broker-credentials.test.ts b/test/deployment-broker-credentials.test.tsnew file mode 100644index 000000000..4a961849e--- /dev/null+++ b/test/deployment-broker-credentials.test.ts@@ -0,0 +1,323 @@+import "./support/auto-fake-sprites.ts";+import { selectDefaultSandbox } from "./support/default-sandbox.ts";++import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import type { AddressInfo } from "node:net";+import { createInsecureTestServer, createServer } from "../src/api/server.ts";+import { buildApp } from "../src/wiring.ts";+import { loadConfig } from "../src/config.ts";+import { CAPABILITY_TTL_MS, CREDENTIAL_BROKER_AUD, mintCapabilityToken } from "../src/auth/capability-token.ts";+import type { BrokerFetch } from "../src/api/credential-broker.ts";+import { decisionsResource } from "../src/api/routes/decisions-resource.ts";+import type { ApiCtx } from "../src/api/routes/route.ts";+import {+ deploymentBrokerCredentialsFromEnv,+ withDeploymentCredentials,+ type DeploymentBrokerCredential,+} from "../src/credentials/deployment-credentials.ts";+import type { ServiceCredentialStore } from "../src/credentials/keychain.ts";+import type { TurnRequest } from "../src/types.ts";+import { testConfig } from "./support/test-config.ts";++const SECRET = "deployment-broker-secret".repeat(3);+const ADMIN = { "content-type": "application/json", "x-admin-actor": "admin-alice@default-org" };+const ORG = "org:default-org";+const KEY = "deployment-supplied-key";++const entry = {+ slug: "search",+ displayName: "Web search",+ host: "api.search.example",+ secretEnv: "SEARCH_KEY",+ injection: { header: "X-API-KEY", scheme: "" },+ allowedMethods: ["POST"],+ allowedPathPrefixes: ["/search"],+ maxRequestsPerDay: 2,+};+const envWith = (list: unknown, extra: Record<string, string> = {}) => ({+ BROKER_CREDENTIALS: JSON.stringify(list),+ SEARCH_KEY: KEY,+ ...extra,+});+const deploymentCreds = (): DeploymentBrokerCredential[] => deploymentBrokerCredentialsFromEnv(envWith([entry]));++test("BROKER_CREDENTIALS parses each entry and reads the secret from the named env var", () => {+ assert.deepEqual(deploymentBrokerCredentialsFromEnv({}), []);+ assert.deepEqual(deploymentCreds(), [+ {+ slug: "search",+ name: "Web search",+ secret: KEY,+ host: "api.search.example",+ injection: { header: "X-API-KEY", scheme: "" },+ allowedMethods: ["POST"],+ allowedPathPrefixes: ["/search"],+ maxRequestsPerDay: 2,+ },+ ]);+ const minimal = deploymentBrokerCredentialsFromEnv(+ envWith([{ slug: "news", host: "news.example.com", secretEnv: "SEARCH_KEY" }]),+ );+ assert.deepEqual(minimal, [{ slug: "news", name: "news", secret: KEY, host: "news.example.com" }]);+});++test("BROKER_CREDENTIALS fails fast on any malformed entry without echoing secrets", () => {+ const cases: Array<[unknown, RegExp, Record<string, string>?]> = [+ [{ nope: 1 }, /must be a JSON array/],+ [["x"], /must be an object/],+ [[{ ...entry, secret: "inline" }], /unknown field "secret"/],+ [[{ ...entry, slug: "Bad Slug" }], /slug must be lowercase/],+ [[entry, entry], /more than once/],+ [[{ ...entry, host: "https://api.search.example/v1" }], /bare lowercase hostname/],+ [[{ ...entry, host: "backend.composio.dev" }], /Composio/],+ [[{ ...entry, secretEnv: "lower" }], /secretEnv/],+ [[{ ...entry, secretEnv: "MISSING_KEY" }], /MISSING_KEY is unset or empty/],+ [[{ ...entry, allowedMethods: ["FETCH"] }], /allowedMethods is invalid/],+ [[{ ...entry, allowedPathPrefixes: ["search"] }], /allowedPathPrefixes is invalid/],+ [[{ ...entry, maxRequestsPerDay: 0 }], /maxRequestsPerDay/],+ [[{ ...entry, maxRequestsPerDay: 1.5 }], /maxRequestsPerDay/],+ [[{ ...entry, injection: { header: "x-qm-actor" } }], /reserved/],+ [[{ ...entry, displayName: "" }], /displayName/],+ ];+ for (const [list, pattern, extra] of cases) {+ assert.throws(+ () => deploymentBrokerCredentialsFromEnv(envWith(list, extra)),+ (error: Error) => pattern.test(error.message) && !error.message.includes(KEY),+ String(pattern),+ );+ }+ assert.throws(() => deploymentBrokerCredentialsFromEnv({ BROKER_CREDENTIALS: "{oops" }), /JSON array/);+ assert.throws(+ () => loadConfig({ ...envWith([{ ...entry, secretEnv: "MISSING_KEY" }]) }),+ /BROKER_CREDENTIALS "search": env var MISSING_KEY/,+ );+});++function memoryStore(): ServiceCredentialStore & { writes: string[] } {+ const writes: string[] = [];+ return {+ writes,+ getServiceCredentialSecret: async (_scope, slug) =>+ slug === "search" || slug === "stored"+ ? {+ slug,+ name: "Stored",+ secret: "db-secret",+ delivery: "broker",+ host: "db.example.com",+ deployments: true,+ enabled: true,+ }+ : null,+ listServiceCredentials: async () =>+ ["search", "stored"].map((slug) => ({+ slug,+ name: "Stored",+ delivery: "broker" as const,+ host: "db.example.com",+ deployments: true,+ enabled: true,+ hasSecret: true,+ updatedAt: 1,+ })),+ setServiceCredential: async (_scope, input) => void writes.push(`set:${input.slug}`),+ setServiceCredentialIfAbsent: async (_scope, input) => (writes.push(`absent:${input.slug}`), 1),+ setServiceCredentialIfCurrent: async (_scope, input) => (writes.push(`current:${input.slug}`), 2),+ deleteServiceCredential: async (_scope, slug) => void writes.push(`delete:${slug}`),+ deleteServiceCredentialIfCurrent: async (_scope, slug) => (writes.push(`deleteIf:${slug}`), true),+ };+}++test("deployment credentials win lookups in their org and reserve their slugs against stored writes", async () => {+ const stored = memoryStore();+ const store = withDeploymentCredentials(stored, ORG, deploymentCreds());++ const listed = await store.listServiceCredentials(ORG);+ assert.deepEqual(+ listed.map((c) => [c.slug, c.source, c.host]),+ [+ ["search", "deployment", "api.search.example"],+ ["stored", undefined, "db.example.com"],+ ],+ );+ assert.doesNotMatch(JSON.stringify(listed), new RegExp(KEY));+ assert.equal(listed[0]!.deployments, false);+ assert.equal((await store.getServiceCredentialSecret(ORG, "search"))?.secret, KEY);+ assert.equal((await store.getServiceCredentialSecret(ORG, "stored"))?.secret, "db-secret");+ assert.equal((await store.getServiceCredentialSecret("org:other", "search"))?.secret, "db-secret");+ assert.equal((await store.listServiceCredentials("org:other")).length, 2);++ const input = { slug: "search", name: "x", host: "evil.example.com", secret: "s" };+ await assert.rejects(store.setServiceCredential(ORG, input), /supplied by this deployment/);+ await assert.rejects(store.setServiceCredentialIfAbsent(ORG, input), /supplied by this deployment/);+ await assert.rejects(store.setServiceCredentialIfCurrent(ORG, input, 1), /supplied by this deployment/);+ await assert.rejects(store.deleteServiceCredential(ORG, "search"), /supplied by this deployment/);+ await assert.rejects(store.deleteServiceCredentialIfCurrent(ORG, "search", 1), /supplied by this deployment/);+ await store.setServiceCredential(ORG, { ...input, slug: "stored" });+ assert.deepEqual(stored.writes, ["set:stored"]);+});++function start(fetchImpl: BrokerFetch, admin = false) {+ const built = buildApp(+ testConfig({+ dataDir: mkdtempSync(join(tmpdir(), "deploy-broker-")),+ signingSecret: SECRET,+ brokerCredentials: deploymentCreds(),+ }),+ );+ const deps = {+ config: built.config,+ serviceCreds: built.serviceCreds,+ acl: built.acl,+ credentialUsage: built.credentialUsage,+ admin: built.admin,+ auditLog: built.auditLog,+ brokerFetch: fetchImpl,+ };+ const server = admin+ ? createInsecureTestServer(built.app, deps)+ : createServer(built.app, { ...deps, signingSecret: SECRET });+ server.listen(0);+ const base = `http://localhost:${(server.address() as AddressInfo).port}`;+ return { base, built, close: () => new Promise<void>((r) => server.close(() => r())) };+}++test("admin settings show a deployment credential read-only and refuse to edit, delete or shadow it", async () => {+ const srv = start(async () => ({ status: 200, text: async () => "{}" }), true);+ try {+ const cfg = (await (await fetch(`${srv.base}/v1/admin/scopes/${ORG}`, { headers: ADMIN })).json()) as {+ serviceCredentials: Array<{ slug: string; source?: string; hasSecret: boolean; maxRequestsPerDay?: number }>;+ };+ const search = cfg.serviceCredentials.find((c) => c.slug === "search");+ assert.equal(search?.source, "deployment");+ assert.equal(search?.hasSecret, true);+ assert.equal(search?.maxRequestsPerDay, 2);+ assert.doesNotMatch(JSON.stringify(cfg), new RegExp(KEY));++ for (const body of [+ { slug: "search", name: "Mine", host: "evil.example.com", secret: "s" },+ { slug: "search", name: "Mine", host: "api.search.example", expectedUpdatedAt: 0 },+ { slug: "search", delete: true, expectedUpdatedAt: 0 },+ ]) {+ const res = await fetch(`${srv.base}/v1/admin/scopes/${ORG}/service-credentials`, {+ method: "PUT",+ headers: ADMIN,+ body: JSON.stringify(body),+ });+ assert.equal(res.status, 409);+ const json = (await res.json()) as { error?: string; message?: string };+ assert.equal(json.error, "deployment_credential");+ assert.match(String(json.message), /supplied by this deployment/);+ }+ } finally {+ await srv.close();+ }+});++test("the broker serves a deployment credential with its pins, injection, usage log and daily cap", async () => {+ const calls: Array<{ url: string; method: string; headers: Record<string, string> }> = [];+ const srv = start(async (url, init) => {+ calls.push({ url, method: init.method, headers: init.headers });+ return { status: 200, contentType: "application/json", text: async () => '{"ok":true}' };+ });+ try {+ const token = await mintCapabilityToken(+ {+ actorId: "U1",+ scopeId: "personal:U1",+ aud: CREDENTIAL_BROKER_AUD,+ credentials: ["search"],+ exp: Date.now() + CAPABILITY_TTL_MS,+ },+ SECRET,+ );+ const call = async (method: string, url: string) => {+ const res = await fetch(`${srv.base}/v1/credentials/broker`, {+ method: "POST",+ headers: { "content-type": "application/json", "x-agent-capability": token },+ body: JSON.stringify({ credential: "search", method, url }),+ });+ return { status: res.status, body: (await res.json()) as { error?: string } };+ };+ assert.equal((await call("POST", "https://evil.example.com/search")).status, 403);+ assert.equal((await call("GET", "https://api.search.example/search")).status, 403);+ assert.equal((await call("POST", "https://api.search.example/admin")).status, 403);+ assert.equal((await call("POST", "https://api.search.example/search?q=1")).status, 200);+ assert.equal((await call("POST", "https://api.search.example/search?q=2")).status, 200);+ const capped = await call("POST", "https://api.search.example/search?q=3");+ assert.equal(capped.status, 429);+ assert.equal(capped.body.error, "daily_cap_reached");+ assert.equal(calls.length, 2);+ assert.equal(calls[0]!.headers["X-API-KEY"], KEY);+ const usage = await srv.built.credentialUsage.list({ slug: "search" });+ assert.equal(usage.filter((u) => u.status === "ok").length, 2);+ assert.equal(usage.filter((u) => u.status === "denied").length, 4);+ } finally {+ await srv.close();+ }+});++const dm = (text: string): TurnRequest => ({+ surface: "test",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: "dm:U1:t1" },+ text,+});++test("org members' agents get a deployment credential by broker only, as if granted org-wide", async () => {+ const built = buildApp(+ testConfig({+ dataDir: mkdtempSync(join(tmpdir(), "deploy-broker-stamp-")),+ maxAttempts: 1,+ signingSecret: SECRET,+ apiBaseUrl: "http://core.internal",+ brokerCredentials: deploymentCreds(),+ }),+ );+ let executed: Record<string, string> | undefined;+ const realRun = built.sandbox.run.bind(built.sandbox);+ built.sandbox.run = (handle, command, opts) => {+ if (command === "echo hi") executed = handle.env;+ return realRun(handle, command, opts);+ };+ await selectDefaultSandbox(built, "U1", "personal:U1");+ const prompt = await built.app.turn(dm("!sysprompt"));+ assert.match(prompt.reply ?? "", /search.*Web search.*api\.search\.example.*POST.*\/search/s);+ const res = await built.app.turn(+ dm(`!execute ${JSON.stringify({ command: "echo hi", credentials: ["service_search"] })}`),+ );+ assert.equal(res.status, "ok", res.reason);+ assert.ok(executed?.AGENT_CREDENTIAL_TOKEN, "expected a broker token");+ assert.doesNotMatch(JSON.stringify(executed), new RegExp(KEY));+});++test("a deployment credential cannot back a decision provider", async () => {+ const saved: unknown[] = [];+ const serviceCreds = withDeploymentCredentials(memoryStore(), ORG, deploymentCreds());+ const ctx = (body: unknown) =>+ ({+ body,+ deps: {+ serviceCreds,+ decisions: {+ saveProvider: async (provider: unknown) => void saved.push(provider),+ providers: async () => [],+ rules: async () => [],+ },+ },+ }) as unknown as ApiCtx;+ const provider = (credential: string) => ({+ provider: { id: "p", protocol: "decisions", baseUrl: "https://api.search.example/search", credential },+ });+ const refused = await decisionsResource.apply!(ctx(provider("search")), { id: "admin" } as never, ORG);+ assert.match(String((refused as { error?: string }).error), /supplied by this deployment/);+ assert.deepEqual(saved, []);+ await decisionsResource.apply!(ctx(provider("stored")), { id: "admin" } as never, ORG);+ assert.equal(saved.length, 1);+ const listing = (await decisionsResource.get!(ctx({}).deps, ORG)) as { credentials: string[] };+ assert.deepEqual(listing.credentials, ["stored"]);+});