YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: verify explicit consent at Grok credential bridge
louis5e6d3712c403
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 7d35feed8..9f1267d84 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 46 git-mined regressions. See+The current app corpus contains 47 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -229,3 +229,18 @@ neighboring passes, quoted assertion diagnostics and error exclusion from scoresAn actual Bun compiler/Vitest setup failure previously certified a false validcontrast; it now becomes an error and invalid verification. This is boundedrecognition, not general compiler attribution or proof of execution isolation.+++The `grok-installer-explicit-consent` case executes the real JavaScript installer+entry point against synthetic encrypted files, the actual decryption routine,+a mocked OS credential command and an in-memory gateway. Fifteen outcomes reject+passive/unsupported actions before file discovery and preserve explicit connect,+idempotent retry, disconnect, unrelated workflows, absent-app handling and safe+errors. Only the installer file from the historical fix is applied as oracle.++Run `bun test evals/coding-agent/calibrate-grok-consent.test.js` for calibrated+parent/reference, equivalent implementation, late refusal, status bypass,+blanket denial, skipped installation/removal and missing-source controls.+This standalone adapter requires macOS; it does not call Keychain or a provider.+It does not verify the native credential-free status cache, browser-cookie+consent, Windows DPAPI, UI interaction or enforced agent isolation.diff --git a/evals/coding-agent/calibrate-grok-consent.test.js b/evals/coding-agent/calibrate-grok-consent.test.jsnew file mode 100644index 000000000..9e4ac65a0--- /dev/null+++ b/evals/coding-agent/calibrate-grok-consent.test.js@@ -0,0 +1,25 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'),source='apps/screenpipe-app-tauri/lib/grokbot-installer.mjs';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='grok-installer-explicit-consent');+const show=ref=>execFileSync('git',['-c','core.commitGraph=false','show',`${ref}:${source}`],{cwd:repo,encoding:'utf8'});+const broken=show(item.base_ref),fixed=show(item.oracle_ref);+const fixture='evals/coding-agent/graders/grok-installer-consent.test.js',grader=readFileSync(join(repo,fixture));+const root=mkdtempSync(join(tmpdir(),'grok-consent-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,text){const cwd=join(root,name);for(const [p,s] of [[source,text],[fixture,grader]]){if(s===null)continue;mkdirSync(dirname(join(cwd,p)),{recursive:true});writeFileSync(join(cwd,p),s);}return spawnSync(process.execPath,['test',fixture],{cwd,encoding:'utf8',timeout:30_000,env:{PATH:dirname(process.execPath)}});}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}+function replace(s,old,next){expect(s.split(old)).toHaveLength(2);return s.replace(old,next);}+test('historical parent fails passive access but preserves explicit actions',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('2 fail');expect(r.stderr).toContain('13 pass');});+test('reference passes all boundary and preserved outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('15 pass');});+test('equivalent parameter name and error wording pass',()=>{const r=grade('equivalent',fixed.replaceAll('input','params').replace('Grok Bot credential access requires an explicit connect or disconnect action.','Explicit authorization is needed.'));expect(r.status).toBe(0);});+test('refusal after probing files cannot pass',()=>fails(grade('late-refusal',replace(fixed,'export async function runInstaller(input) {',"export async function runInstaller(input) { if (input.action === 'status') { existsSync(appDataPath(input.home, process.platform, {})); throw Error('Refused'); }"))));+test('allowing status again is rejected',()=>fails(grade('status-bypass',replace(fixed,'if (!["connect", "disconnect"].includes(input.action))','if (!["status", "connect", "disconnect"].includes(input.action))'))));+test('blanket denial cannot replace consent handling',()=>fails(grade('deny-all',replace(fixed,'export async function runInstaller(input) {',"export async function runInstaller(input) { throw Error('Refused');"))));+test('claiming connected without installation is rejected',()=>fails(grade('fake-connect',replace(fixed,'export async function runInstaller(input) {',"export async function runInstaller(input) { if (input.action === 'connect') return { detected: true, connected: true };"))));+test('claiming disconnected without removal is rejected',()=>fails(grade('fake-disconnect',replace(fixed,'export async function runInstaller(input) {',"export async function runInstaller(input) { if (input.action === 'disconnect') return { detected: true, connected: false };"))));+test('missing source is a setup failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex e8b4d8c74..a151a7414 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-19.3",+ "dataset_version": "2026-09-19.4","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1484,6 +1484,38 @@],"command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 \"${BUN_BIN:-bun}\" run test -- src/eval-team-auth.test.ts"}+ },+ {+ "id": "grok-installer-explicit-consent",+ "name": "Require explicit action before the Grok installer reads credentials",+ "source": {+ "kind": "git_regression",+ "fix_commit": "b6b18e4fa628e53bbf7eccef974faf04a07a95cc"+ },+ "base_ref": "b247eb3c674af026fe28c628a098e9f8c748c8a5",+ "oracle_ref": "b6b18e4fa628e53bbf7eccef974faf04a07a95cc",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/grokbot-installer.mjs"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "privacy",+ "credentials",+ "consent"+ ],+ "prompt": "The Grok installer credential bridge currently accepts passive status requests and reads another application's credential files. Require an explicit connect or disconnect action before discovering the application files, reading protected credentials or contacting its gateway. Reject status and all other unsupported actions before those effects, whether or not the app is installed. Preserve explicit connect and disconnect: use the existing saved gateway credential only for its authorized gateway requests, verify installation, retry without creating duplicate skills, remove only this device's owned skill on disconnect, and preserve unrelated user workflows. Explicit requests with no installed app should report not detected/disconnected without credential or network access. Credential-command and network failures must not disclose credentials in returned errors. Verify the actual JavaScript installer entry point using synthetic files and provider/OS ports. Native status-cache behavior, UI consent and Windows credential handling are outside this task.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/grok-installer-consent.test.js",+ "destination_path": "evals/coding-agent/graders/grok-installer-consent.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/grok-installer-consent.test.js"+ }}]}diff --git a/evals/coding-agent/graders/grok-installer-consent.test.js b/evals/coding-agent/graders/grok-installer-consent.test.jsnew file mode 100644index 000000000..9711abf85--- /dev/null+++ b/evals/coding-agent/graders/grok-installer-consent.test.js@@ -0,0 +1,87 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, beforeEach, expect, mock, test } from 'bun:test';+import * as fs from 'node:fs';+import { join } from 'node:path';+import { tmpdir } from 'node:os';+import { createCipheriv, pbkdf2Sync } from 'node:crypto';++// Execute the real bridge and crypto with synthetic files. Never invoke a real+// credential command or network provider. This fixture currently targets macOS.+if (process.platform !== 'darwin') throw Error('This synthetic credential adapter requires macOS');+const realRead=fs.readFileSync, realExists=fs.existsSync;+const root=fs.mkdtempSync(join(tmpdir(),'grok-consent-'));+const home=join(root,'home'), dir=join(home,'Library','Application Support','Grok Bot');+const password='synthetic-safe-storage-password', token='synthetic-gateway-token';+const originalFetch=globalThis.fetch;+let reads,probes,commands,requests,rows,sequence,commandError,networkError;+mock.module('node:fs',()=>({...fs,+ readFileSync:(path,...args)=>{reads.push(String(path));return realRead(path,...args);},+ existsSync:path=>{probes.push(String(path));return realExists(path);},+}));+mock.module('node:child_process',()=>({execFileSync:(file,args)=>{+ commands.push({file,args});+ if(commandError)throw Error(token);+ if(file!=='/usr/bin/security'||!args.includes('Grok Bot Safe Storage'))throw Error('Unexpected credential adapter operation');+ return password;+}}));+const {runInstaller}=await import('../../../apps/screenpipe-app-tauri/lib/grokbot-installer.mjs');+const input=action=>({action,home,bun:'/synthetic/bun',dataDir:'/synthetic/recordings',port:3137,skill:'Synthetic public API reference.'});+const unrelated={id:'user-owned',source:'workflow',sourceRef:'https://example.invalid/user',name:'User workflow',body:'Preserve my workflow'};+function installFiles(){+ fs.mkdirSync(dir,{recursive:true});+ const cipher=createCipheriv('aes-128-cbc',pbkdf2Sync(password,'saltysalt',1003,16,'sha1'),Buffer.alloc(16,32));+ const clear=JSON.stringify({baseUrl:'https://synthetic.cursorvm.com/',token});+ const encrypted=Buffer.concat([Buffer.from('v10'),cipher.update(clear),cipher.final()]).toString('base64');+ fs.writeFileSync(join(dir,'sand-secrets.json'),JSON.stringify({'cursor-accounts':JSON.stringify({active:'fixture-account'})}));+ fs.writeFileSync(join(dir,'gateway-descriptor.json'),JSON.stringify({version:2,entries:{'fixture-account':{encrypted,savedAtMs:Date.now()-100}}}));+}+beforeEach(()=>{+ reads=[];probes=[];commands=[];requests=[];sequence=0;rows=[structuredClone(unrelated)];commandError=false;networkError=false;installFiles();+ globalThis.fetch=async(url,init)=>{+ requests.push({url:String(url),init});if(networkError)throw Error(token);+ expect(String(url).startsWith('https://synthetic.cursorvm.com/api/')).toBe(true);+ expect(init.method).toBe('POST');expect(init.redirect).toBe('error');+ expect(new Headers(init.headers).get('authorization')).toBe(`Bearer ${token}`);+ expect(init.body).not.toContain(token);+ const action=new URL(url).pathname.split('/').at(-1),args=JSON.parse(init.body);+ if(action==='listAgents')return Response.json([{id:'bot',isGroup:false}]);+ if(action==='getAgentWorkflows')return Response.json(rows);+ if(action==='createAgentWorkflow')rows.push({...args.spec,id:`owned-${++sequence}`,source:'workflow'});+ else if(action==='updateAgentWorkflow')rows=rows.map(row=>row.id===args.workflowId?{...row,...args.spec}:row);+ else if(action==='deleteAgentWorkflow')rows=rows.filter(row=>row.id!==args.workflowId);+ else throw Error('Unexpected synthetic gateway action');+ return Response.json(rows);+ };+});+afterAll(()=>{globalThis.fetch=originalFetch;fs.rmSync(root,{recursive:true,force:true});});+function noAccess(){expect(probes).toEqual([]);expect(reads).toEqual([]);expect(commands).toEqual([]);expect(requests).toEqual([]);expect(rows).toEqual([unrelated]);}+async function explicit(action){let result,failure;try{result=await runInstaller(input(action));}catch(error){failure=error;}expect(failure).toBeUndefined();return result;}+function safe(value){expect(JSON.stringify(value)).not.toContain(token);expect(JSON.stringify(value)).not.toContain(password);}+for(const action of ['status','automatic','',undefined,null,'CONNECT',true,{}])test(`non-explicit action ${JSON.stringify(action)} rejects before file discovery`,async()=>{+ await expect(runInstaller(input(action))).rejects.toThrow();noAccess();+});+test('passive status does not probe even when the app is absent',async()=>{+ fs.rmSync(dir,{recursive:true});await expect(runInstaller(input('status'))).rejects.toThrow();noAccess();+});+test('explicit connect verifies installation and retries without duplicates',async()=>{+ const first=await explicit('connect');expect(first.connected).toBe(true);safe(first);+ expect(reads).toContain(join(dir,'sand-secrets.json'));expect(reads).toContain(join(dir,'gateway-descriptor.json'));expect(commands).toHaveLength(1);+ expect(rows).toHaveLength(2);expect(rows[0]).toEqual(unrelated);safe(rows);+ requests=[];const second=await explicit('connect');expect(second.connected).toBe(true);safe(second);+ expect(rows).toHaveLength(2);expect(requests.every(r=>['listAgents','getAgentWorkflows'].includes(new URL(r.url).pathname.split('/').at(-1)))).toBe(true);+});+test('explicit disconnect removes owned installation and preserves user content',async()=>{+ await explicit('connect');requests=[];+ const result=await explicit('disconnect');expect(result.connected).toBe(false);safe(result);+ expect(rows).toEqual([unrelated]);expect(requests.some(r=>String(r.url).endsWith('/deleteAgentWorkflow'))).toBe(true);+});+for(const action of ['connect','disconnect'])test(`explicit ${action} with absent app needs no credentials or network`,async()=>{+ fs.rmSync(dir,{recursive:true});const result=await explicit(action);expect(result).toMatchObject({detected:false,connected:false});safe(result);+ expect(reads).toEqual([]);expect(commands).toEqual([]);expect(requests).toEqual([]);expect(rows).toEqual([unrelated]);+});+for(const kind of ['credential','network'])test(`explicit ${kind} failure reports no credentials and preserves unrelated workflow`,async()=>{+ commandError=kind==='credential';networkError=kind==='network';+ let failure;try{await runInstaller(input('connect'));}catch(error){failure=error;}+ expect(failure).toBeInstanceOf(Error);safe({message:failure.message});expect(rows).toEqual([unrelated]);+});