Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: verify running MCP enterprise credential reload

louis02d94c48a644
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 4568b04d7..5315e0c62 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 45 git-mined regressions. See
+The current app corpus contains 46 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -194,3 +194,26 @@ This is input provenance, not a complete reproducibility or isolation guarantee.
Dependency directories, compilers, external services, ambient environment and
sandbox enforcement are not attested by these hashes. Do not use a matching
fingerprint alone to claim matched agent trials or unchanged product behavior.
+
+
+## Running MCP enterprise credentials
+
+`mcp-team-credential-reload` builds and starts the real historical MCP server,
+connects a real SDK stdio client, and observes synthetic loopback HTTP requests
+while changing a temporary enterprise settings file. Seven scenarios cover token
+replacement, matching gateway rotation, first-time configuration, clear/delete/
+malformed recovery, override precedence, expired-token guidance without secret
+exposure, and preservation of local tool listings. Telemetry is disabled and
+local API credentials are synthetic. Only the two production source changes
+are applied by the oracle; builds are created inside each grading workspace.
+
+Run `bun test evals/coding-agent/calibrate-mcp-team-reload.test.js` with the MCP
+package dependencies installed. Eight controls retain the parent/reference
+contrast, reject an unused-helper fix, cached settings, stale gateway pairing
+and blanket denial, accept equivalent naming, and distinguish a missing-source
+build failure. These are deterministic corpus/grader tests, not model trials.
+
+The fixture tests sequential saved-file changes and real local process/protocol
+behavior. It does not verify concurrent file writes, native desktop persistence,
+external enterprise gateways, token cryptography or the harness's filesystem
+isolation. Dependencies are exposed only during grading, after the trajectory.
diff --git a/evals/coding-agent/calibrate-mcp-team-reload.test.js b/evals/coding-agent/calibrate-mcp-team-reload.test.js
new file mode 100644
index 000000000..1861dbf6a
--- /dev/null
+++ b/evals/coding-agent/calibrate-mcp-team-reload.test.js
@@ -0,0 +1,49 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='mcp-team-credential-reload');
+const pkg='packages/screenpipe-mcp',index=`${pkg}/src/index.ts`,config=`${pkg}/src/team-config.ts`;
+const show=(ref,p)=>execFileSync('git',['-c','core.commitGraph=false','show',`${ref}:${p}`],{cwd:repo,encoding:'utf8'});
+const fixed={[index]:show(item.oracle_ref,index),[config]:show(item.oracle_ref,config)};
+const broken={[index]:show(item.base_ref,index),[config]:show(item.base_ref,config)};
+const root=mkdtempSync(join(tmpdir(),'mcp-team-reload-'));
+const template=join(root,'template');mkdirSync(template);
+const archive=execFileSync('git',['archive',item.base_ref,pkg],{cwd:repo,maxBuffer:64*1024*1024});
+execFileSync('tar',['-xf','-','-C',template],{input:archive});
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,sources){
+ const cwd=join(root,name);cpSync(template,cwd,{recursive:true});
+ for(const [p,s] of Object.entries(sources)){
+ if(s===null)rmSync(join(cwd,p));else writeFileSync(join(cwd,p),s);
+ }
+ writeFileSync(join(cwd,pkg,'src/eval-team-auth.test.ts'),readFileSync(join(import.meta.dir,'graders/mcp-team-reload.fixture.ts')));
+ symlinkSync(join(repo,pkg,'node_modules'),join(cwd,pkg,'node_modules'),'dir');
+ const r=spawnSync(process.execPath,['run','test','--','src/eval-team-auth.test.ts'],{cwd:join(cwd,pkg),encoding:'utf8',timeout:45_000,env:{PATH:process.env.PATH,HOME:cwd,TZ:'UTC',CI:'true',NO_COLOR:'1',FORCE_COLOR:'0',SCREENPIPE_DISABLE_TELEMETRY:'1'}});
+ for(const key of ['stdout','stderr'])r[key]=r[key]?.replace(/\x1b\[[0-9;]*m/g,'');
+ return r;
+}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(r.stdout).toMatch(/Tests\s+\d+ failed/);expect(r.stderr).not.toContain('Cannot find module');}
+function replace(s,old,next){expect(s.split(old)).toHaveLength(2);return s.replace(old,next);}
+test('broken parent reaches intended credential and gateway failures',()=>{const r=grade('parent',broken);fails(r);expect(r.stdout).toContain('6 failed | 1 passed (7)');},60_000);
+test('historical reference passes real running-process outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stdout).toContain('7 passed');},60_000);
+test('unused correct config helper beside stale caller is rejected',()=>fails(grade('helper-only',{...broken,[config]:fixed[config]})),60_000);
+test('equivalent helper and request snapshot renaming passes',()=>{
+ const sources=Object.fromEntries(Object.entries(fixed).map(([p,s])=>[p,s.replaceAll('discoverTeamConfig','loadCurrentTeamSettings').replaceAll('teamConfig','requestSettings')]));expect(grade('renamed',sources).status).toBe(0);
+},60_000);
+test('one boot-time snapshot cannot masquerade as current request state',()=>{
+ const s=fixed[index].replace('// Initialize server','const cachedTeamConfig = discoverTeamConfig(teamApiOverride);\n// Initialize server').replaceAll('const { token } = discoverTeamConfig(teamApiOverride);','const { token } = cachedTeamConfig;').replaceAll('const teamConfig = discoverTeamConfig(teamApiOverride);','const teamConfig = cachedTeamConfig;');
+ fails(grade('cached',{...fixed,[index]:s}));
+},60_000);
+test('blanket team denial fails preserved working access',()=>fails(grade('deny-all',{...fixed,[index]:replace(fixed[index],'if (!teamConfig.token) {','if (true) {')})),60_000);
+test('new token paired with stale gateway is rejected',()=>{
+ const s=fixed[index].replace('// Initialize server','const cachedTeamGateway = discoverTeamConfig(teamApiOverride).apiBase;\n// Initialize server').replace('`${config.apiBase}${p}`','`${cachedTeamGateway}${p}`');
+ fails(grade('gateway-stale',{...fixed,[index]:s}));
+},60_000);
+test('missing config source is a build error rather than behavior evidence',()=>{
+ const r=grade('missing-source',{...fixed,[config]:null});expect(r.status).toBe(1);expect(r.stdout).toContain('skipped');expect(r.stderr).toContain('Could not resolve');expect(r.stderr).not.toContain('AssertionError');
+},60_000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index cee74bc97..e8b4d8c74 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-19.2",
+ "dataset_version": "2026-09-19.3",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1445,6 +1445,45 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-billing-capacity.test.js"
}
+ },
+ {
+ "id": "mcp-team-credential-reload",
+ "name": "Reload enterprise credentials in the running MCP server",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "909c9360a4379896df49507140017e9989d41428"
+ },
+ "base_ref": "e681fc3f44f44fc377c771988814db00021b2e84",
+ "oracle_ref": "909c9360a4379896df49507140017e9989d41428",
+ "oracle_paths": [
+ "packages/screenpipe-mcp/src/index.ts",
+ "packages/screenpipe-mcp/src/team-config.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "mcp",
+ "enterprise",
+ "identity"
+ ],
+ "prompt": "The desktop can save a replacement enterprise token or team gateway while its MCP server keeps running, but team requests continue using stale credentials. Make the next team-tool request and tool listing reflect the current saved configuration without restarting the process, including credentials saved for the first time. Keep token and gateway from the same configuration snapshot. Clearing, deleting or corrupting the saved configuration must hide unavailable team tools and reject their calls before any team request; restoring valid settings must recover. Preserve local tools and explicit environment-token overrides, and preserve gateway precedence: command-line override, environment override, saved gateway, then hosted default. An expired explicit token must stay authoritative and give recovery guidance identifying the environment override and enterprise.json settings without exposing the credential. Verify observable requests and results through the running MCP process using synthetic tokens and local test servers.",
+ "agent_timeout_seconds": 1200,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/screenpipe-mcp/node_modules",
+ "destination_path": "packages/screenpipe-mcp/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/mcp-team-reload.fixture.ts",
+ "destination_path": "packages/screenpipe-mcp/src/eval-team-auth.test.ts"
+ }
+ ],
+ "command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 \"${BUN_BIN:-bun}\" run test -- src/eval-team-auth.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/mcp-team-reload.fixture.ts b/evals/coding-agent/graders/mcp-team-reload.fixture.ts
new file mode 100644
index 000000000..c229edb71
--- /dev/null
+++ b/evals/coding-agent/graders/mcp-team-reload.fixture.ts
@@ -0,0 +1,188 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { afterEach, beforeAll, describe, expect, it } from "vitest";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { execFileSync } from "node:child_process";
+import * as fs from "node:fs";
+import { createServer, type Server } from "node:http";
+import * as os from "node:os";
+import * as path from "node:path";
+
+const PKG_ROOT = path.resolve(__dirname, "..");
+const clients: Client[] = [];
+const servers: Server[] = [];
+const homes: string[] = [];
+
+beforeAll(() => {
+ execFileSync("bun", ["run", "build"], { cwd: PKG_ROOT, timeout: 120_000 });
+}, 120_000);
+
+afterEach(async () => {
+ await Promise.all(clients.splice(0).map((client) => client.close()));
+ await Promise.all(servers.splice(0).map((server) => new Promise<void>((resolve) => {
+ server.close(() => resolve());
+ server.closeAllConnections();
+ })));
+ for (const home of homes.splice(0)) fs.rmSync(home, { recursive: true, force: true });
+});
+
+function newHome(): string {
+ const home = fs.mkdtempSync(path.join(os.tmpdir(), "sp-mcp-team-auth-"));
+ homes.push(home);
+ fs.mkdirSync(path.join(home, ".screenpipe"));
+ return home;
+}
+
+function saveConfig(home: string, token: string, apiBase: string): void {
+ fs.writeFileSync(path.join(home, ".screenpipe", "enterprise.json"), JSON.stringify({
+ team_api_token: token,
+ gateway_url: apiBase,
+ }));
+}
+
+async function teamApi(validToken: string) {
+ const requests: { url: string; authorization: string | undefined }[] = [];
+ const server = createServer((req, res) => {
+ requests.push({ url: req.url || "", authorization: req.headers.authorization });
+ res.setHeader("content-type", "application/json");
+ if (req.headers.authorization !== `Bearer ${validToken}`) {
+ res.writeHead(401);
+ res.end(JSON.stringify({ error: "token expired" }));
+ } else {
+ res.end(JSON.stringify({ devices: [{ device_id: "test-device" }] }));
+ }
+ });
+ servers.push(server);
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const address = server.address();
+ if (!address || typeof address === "string") throw new Error("missing test server address");
+ return { apiBase: `http://127.0.0.1:${address.port}/api/enterprise/v1`, requests };
+}
+
+async function startMcp(home: string, env: Record<string, string> = {}, flags: string[] = []) {
+ const client = new Client({ name: "team-auth-test", version: "1.0.0" });
+ clients.push(client);
+ await client.connect(new StdioClientTransport({
+ command: process.execPath,
+ args: [path.join(PKG_ROOT, "dist", "cli.js"), ...flags],
+ env: {
+ HOME: home,
+ USERPROFILE: home,
+ SCREENPIPE_DISABLE_TELEMETRY: "1",
+ SCREENPIPE_LOCAL_API_KEY: "sp-test-local-key",
+ SCREENPIPE_API_URL: "http://127.0.0.1:59999",
+ ...env,
+ },
+ stderr: "pipe",
+ }));
+ return client;
+}
+
+const devices = (client: Client) => client.callTool({ name: "team-devices", arguments: {} });
+const hasTeamTools = async (client: Client) =>
+ (await client.listTools()).tools.some((tool) => tool.name === "team-devices");
+
+describe("running MCP team credentials", () => {
+ it("uses a replacement file token on the next call and stops using a cleared token", async () => {
+ const home = newHome();
+ const api = await teamApi("sk_ent_current");
+ saveConfig(home, "sk_ent_expired", api.apiBase);
+ const client = await startMcp(home);
+ expect((await devices(client)).isError).toBe(true);
+
+ saveConfig(home, "sk_ent_current", api.apiBase);
+ expect((await devices(client)).isError).not.toBe(true);
+ expect(api.requests.map((req) => req.authorization)).toEqual([
+ "Bearer sk_ent_expired", "Bearer sk_ent_current",
+ ]);
+
+ saveConfig(home, "", api.apiBase);
+ expect(await hasTeamTools(client)).toBe(false);
+ expect((await devices(client)).isError).toBe(true);
+ expect(api.requests).toHaveLength(2);
+
+ saveConfig(home, "sk_ent_current", api.apiBase);
+ expect(await hasTeamTools(client)).toBe(true);
+ expect((await devices(client)).isError).not.toBe(true);
+ });
+
+ it("loads the matching gateway when the file token and gateway change together", async () => {
+ const home = newHome();
+ const first = await teamApi("sk_ent_first_org");
+ const second = await teamApi("sk_ent_second_org");
+ saveConfig(home, "sk_ent_first_org", first.apiBase);
+ const client = await startMcp(home);
+ expect((await devices(client)).isError).not.toBe(true);
+
+ saveConfig(home, "sk_ent_second_org", second.apiBase);
+ expect((await devices(client)).isError).not.toBe(true);
+ expect(first.requests).toEqual([{ url: "/api/enterprise/v1/devices", authorization: "Bearer sk_ent_first_org" }]);
+ expect(second.requests).toEqual([{ url: "/api/enterprise/v1/devices", authorization: "Bearer sk_ent_second_org" }]);
+ });
+
+ it("preserves an explicit expired token and explains how to remove the stale override", async () => {
+ const home = newHome();
+ const api = await teamApi("sk_ent_current");
+ saveConfig(home, "sk_ent_current", api.apiBase);
+ const client = await startMcp(home, { SCREENPIPE_ENTERPRISE_TOKEN: "sk_ent_expired_override" });
+ const result = await devices(client);
+ expect(result.isError).toBe(true);
+ const message = JSON.stringify(result.content);
+ expect(message).toContain("SCREENPIPE_ENTERPRISE_TOKEN");
+ expect(message).toContain("enterprise.json");
+ expect(message).not.toContain("sk_ent_expired_override");
+ expect(api.requests).toEqual([{ url: "/api/enterprise/v1/devices", authorization: "Bearer sk_ent_expired_override" }]);
+ });
+});
+
+
+describe("preserved team configuration boundaries", () => {
+ it("picks up first-time credentials without changing local tools", async () => {
+ const home = newHome();
+ const api = await teamApi("sk_ent_first");
+ const client = await startMcp(home);
+ const localTools = (await client.listTools()).tools.map(t => t.name);
+ expect(localTools.length).toBeGreaterThan(0);
+ expect(await hasTeamTools(client)).toBe(false);
+ expect((await devices(client)).isError).toBe(true);
+ expect(api.requests).toHaveLength(0);
+ saveConfig(home, "sk_ent_first", api.apiBase);
+ expect(await hasTeamTools(client)).toBe(true);
+ expect((await devices(client)).isError).not.toBe(true);
+ expect((await client.listTools()).tools.filter(t => !t.name.startsWith("team-")).map(t => t.name)).toEqual(localTools);
+ expect(api.requests).toEqual([{url:"/api/enterprise/v1/devices",authorization:"Bearer sk_ent_first"}]);
+ });
+
+ for (const kind of ["deleted", "malformed"] as const) {
+ it(`${kind} saved credentials stop team requests and can be restored`, async () => {
+ const home = newHome();
+ const api = await teamApi("sk_ent_saved");
+ saveConfig(home, "sk_ent_saved", api.apiBase);
+ const client = await startMcp(home);
+ expect((await devices(client)).isError).not.toBe(true);
+ const file = path.join(home,".screenpipe","enterprise.json");
+ if(kind === "deleted") fs.unlinkSync(file); else fs.writeFileSync(file,"{ invalid json");
+ expect(await hasTeamTools(client)).toBe(false);
+ expect((await devices(client)).isError).toBe(true);
+ expect(api.requests).toHaveLength(1);
+ saveConfig(home, "sk_ent_saved", api.apiBase);
+ expect((await devices(client)).isError).not.toBe(true);
+ expect(api.requests).toHaveLength(2);
+ });
+ }
+
+ it("keeps the explicit gateway override when the saved gateway changes", async () => {
+ const home = newHome();
+ const explicit = await teamApi("sk_ent_override");
+ const ignored = await teamApi("sk_ent_override");
+ saveConfig(home,"sk_ent_override",ignored.apiBase);
+ const client = await startMcp(home,{SCREENPIPE_TEAM_API_URL:ignored.apiBase},["--team-api-url",explicit.apiBase]);
+ expect((await devices(client)).isError).not.toBe(true);
+ saveConfig(home,"sk_ent_override",ignored.apiBase+"/changed");
+ expect((await devices(client)).isError).not.toBe(true);
+ expect(explicit.requests).toHaveLength(2);
+ expect(ignored.requests).toHaveLength(0);
+ });
+});