Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: verify independent workflow sharing consent on entry

louis787902327262
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index cf13f8ef1..9440750ba 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,18 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-workflow-sharing-entry-consent` renders the real workspace prompt and sharing
+controls with synthetic settings, consent-service and native-token ports. Nine
+outcomes cover independent entry, backend readiness, account hydration/switching,
+notice persistence, reconnects and preserved explicit consent and schedule actions.
+The parent fails six entry outcomes and preserves three; the reference and current
+source pass nine. Run `bun test evals/coding-agent/calibrate-sharing-entry.test.js`
+for eleven controls, including equivalent conditions, unused correct code, local
+consent granted by skipping, missing account notices and lost positive consent.
+Missing source remains a setup error. Fixtures and dependency links are installed
+only at grading. This does not establish server enforcement, native durability,
+real uploads, execution isolation or model improvement.
+
`app-workflow-voice-immutable-response` runs the actual gateway and Durable Object
in the existing local Workers harness, with local D1 and synthetic account and
voice-provider replies. The parent returns 500 instead of the intended 400;
diff --git a/evals/coding-agent/calibrate-sharing-entry.test.js b/evals/coding-agent/calibrate-sharing-entry.test.js
new file mode 100644
index 000000000..4a32de13b
--- /dev/null
+++ b/evals/coding-agent/calibrate-sharing-entry.test.js
@@ -0,0 +1,43 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-workflow-sharing-entry-consent');
+const source=`${app}/components/workflows/workflow-tasks-prompt.tsx`;
+const root=mkdtempSync(join(tmpdir(),'sharing-entry-calibration-')), archives=new Map();
+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function replace(cwd,file,from,to){const p=join(cwd,file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);
+ if(!archives.has(ref)){
+ const children=execFileSync('git',['ls-tree','--name-only',`${ref}:${app}`],{cwd:repo,encoding:'utf8'}).trim().split('\n');
+ const paths=children.filter(n=>!['src-tauri','public','e2e','.e2e'].includes(n)&&!n.startsWith('.env')).map(n=>`${app}/${n}`);
+ archives.set(ref,execFileSync('git',['archive',ref,...paths],{cwd:repo,maxBuffer:128*1024*1024}));
+ }
+ execFileSync('tar',['-x','-C',cwd],{input:archives.get(ref)});
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);
+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});
+ const kind=classifyGraderError(r);
+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message,error_kind:kind},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return {...r,kind};
+}
+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('9 passed');expect(r.kind).toBeNull();}
+function fails(r){expect(r.status).toBe(1);expect(r.kind).toBeNull();expect(r.stdout).toMatch(/Tests\s+[1-9]\d* failed/);expect(r.stderr).not.toContain('TypeError:');}
+test('parent fails intended entry outcomes while preserved controls pass',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('3 passed');},60000);
+test('reference passes',()=>passes(grade('reference')),60000);
+test('current passes',()=>passes(grade('current','HEAD')),60000);
+test('equivalent active condition passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,source,'if (active && account &&','if (account && active &&'))),60000);
+test('unused fixed component cannot repair parent',()=>fails(grade('unused',item.base_ref,cwd=>writeFileSync(join(cwd,source+'.unused.txt'),execFileSync('git',['show',`${item.oracle_ref}:${source}`],{cwd:repo})))),60000);
+test('recorder-gated choice fails',()=>fails(grade('recorder',item.oracle_ref,cwd=>replace(cwd,source,'if (active && account &&','if (active && backendReady && account &&'))),60000);
+test('skip without account notice fails',()=>fails(grade('skip',item.oracle_ref,cwd=>replace(cwd,source,'if (account) void updateSettings','if (false) void updateSettings'))),60000);
+test('disabled positive consent fails preserved behavior',()=>fails(grade('grant',item.oracle_ref,cwd=>replace(cwd,`${app}/components/workflows/workflow-sharing-controls.tsx`,'onClick={() => void change(true)}','onClick={() => {}}'))),60000);
+test('missing component is infrastructure failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,source)));expect(r.status).toBe(1);expect(r.kind).toBe('vitest_collection_error');},60000);
+test('skip cannot silently grant local sharing',()=>fails(grade('skip-grant',item.oracle_ref,cwd=>replace(cwd,source,'void updateSettings({ workflowSharingPromptSeen:','void updateSettings({ workflowSharing: { accountId: account, epoch: "forged", priorBackend: "local" }, workflowSharingPromptSeen:'))),60000);
+test('a different account notice cannot suppress the current choice',()=>fails(grade('global-notice',item.oracle_ref,cwd=>replace(cwd,source,'value.workflowSharingPromptSeen?.[account] !== SHARING_NOTICE_VERSION','!Object.values(value.workflowSharingPromptSeen || {}).includes(SHARING_NOTICE_VERSION)'))),60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 5baecec95..299bf4f38 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4379,6 +4379,49 @@
],
"command": "cd packages/ai-gateway && WRANGLER_SEND_METRICS=false \"${BUN_BIN:-bun}\" test src/test/eval-workflow-voice-response.test.ts --timeout 45000"
}
+ },
+ {
+ "id": "app-workflow-sharing-entry-consent",
+ "name": "Offer independent account-scoped workflow sharing consent",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "05b3f08ae1574c728ba7af952b12bdc954df1e06"
+ },
+ "base_ref": "97adca652c426d95bcb192c76013852158446804",
+ "oracle_ref": "05b3f08ae1574c728ba7af952b12bdc954df1e06",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/workflows/workflow-tasks-prompt.tsx"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/workflows/workflow-tasks-prompt.tsx",
+ "apps/screenpipe-app-tauri/components/workflows/workflow-sharing-controls.tsx"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "privacy",
+ "consent",
+ "account-boundary"
+ ],
+ "prompt": "Entering Workflows does not offer the optional sharing choice when automatic updates are off or the recorder is still connecting. Make the real workspace prompt and sharing controls offer an authenticated account its unseen sharing notice independently of recorder readiness and update allowance. Merely opening or skipping must not grant sharing, change redaction/recording preferences, or enable schedules. Skipping records the current notice for that account while preserving other accounts; returning to Workflows must not repeat it. Wait for account hydration, offer a fresh choice when switching to an unseen account, and close on sign-out. An unanswered choice must survive recorder disconnect/reconnect. Preserve no prompt or service work while Workflows is inactive, and no prompt for an already-seen account. Preserve explicit successful sharing through the existing consent service, including both sharing and training permission, account/epoch persistence and cloud redaction without changing recording preferences. An unavailable account must not be allowed to grant sharing and must still be able to skip. After skipping, normal schedule confirmation must enable updates without reopening sharing or granting it. Keep public component and service contracts compatible; preserve existing accessible action labels. No particular hook structure is required. Tests execute both real React components with synthetic settings, consent-service and native-token ports. Server consent enforcement, durable native settings and real data upload are outside this task.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 90,
+ "fixtures": [
+ {
+ "local_path": "graders/sharing-entry.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/components/workflows/eval-sharing-entry.test.tsx"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/workflows/eval-sharing-entry.test.tsx --maxWorkers=1 --minWorkers=1"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/sharing-entry.fixture.tsx.txt b/evals/coding-agent/graders/sharing-entry.fixture.tsx.txt
new file mode 100644
index 000000000..ed123e127
--- /dev/null
+++ b/evals/coding-agent/graders/sharing-entry.fixture.tsx.txt
@@ -0,0 +1,96 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import React from 'react';
+import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { WorkflowTasksPrompt } from './workflow-tasks-prompt';
+const f = vi.hoisted(() => ({ settings: {} as any, update: vi.fn(), request: vi.fn(), token: vi.fn(), stop: vi.fn() }));
+vi.mock('@/lib/hooks/use-settings', () => ({ useSettings: () => ({ settings: f.settings, updateSettings: f.update }) }));
+vi.mock('@/lib/utils/tauri', () => ({ commands: { getCloudToken: f.token } }));
+vi.mock('@/lib/trajectories/collector', () => ({ SHARING_NOTICE_VERSION: '2026-09-23', sharingRequest: f.request, trajectoryCollector: { stop: f.stop } }));
+vi.mock('@/lib/workflows/scheduled-discovery', () => ({ loadWorkflowTaskSetup: vi.fn(), enableWorkflowTask: vi.fn(), disableWorkflowTasks: vi.fn() }));
+const notice = '2026-09-23';
+function status(extra = {}) { return { available: true, noticeVersion: notice, acceptedNoticeVersion: null, accountId: f.settings.user?.id, sharing: false, training: false, epoch: null, revision: 1, ...extra }; }
+beforeEach(() => {
+ vi.clearAllMocks();
+ f.settings = { user: { id: 'synthetic-a' }, workflowSharingPromptSeen: {}, workflowSharing: null, piiBackend: 'local', disabled: true };
+ f.update.mockImplementation(async (patch: any) => { f.settings = { ...f.settings, ...patch }; });
+ f.token.mockImplementation(async () => f.settings.user ? `synthetic-token-${f.settings.user.id}` : null);
+ f.request.mockImplementation(async (method: string) => status(method === 'PUT' ? { sharing: true, training: true, acceptedNoticeVersion: notice, epoch: 'synthetic-epoch' } : {}));
+});
+afterEach(cleanup);
+function service(enabled = false) {
+ return { load: vi.fn(async () => ({ enabled, title: 'Synthetic updates', schedule: 'Hourly', tasks: [{ name: 'synthetic-task', enabled }] })), enable: vi.fn(async () => { enabled = true; }), disable: vi.fn(async () => { enabled = false; }) };
+}
+const dialog = () => screen.findByRole('dialog', { name: 'Help improve Screenpipe' });
+const skip = () => fireEvent.click(screen.getByRole('button', { name: 'Not now' }));
+const toggle = () => screen.getByRole('switch', { name: 'Automatic updates' });
+const grants = () => f.request.mock.calls.filter(([method]) => method !== 'GET');
+async function flush() { await act(async () => { await Promise.resolve(); }); }
+it('offers a real optional sharing choice on entry while schedules are off', async () => {
+ const tasks = service(); render(<WorkflowTasksPrompt active tasks={tasks} />);
+ expect(await dialog()).toBeVisible();
+ await waitFor(() => expect(screen.getByRole('button', { name: 'Allow sharing' })).toBeEnabled());
+ expect(f.update).not.toHaveBeenCalled(); expect(grants()).toEqual([]);
+ expect(tasks.enable).not.toHaveBeenCalled(); expect(tasks.disable).not.toHaveBeenCalled();
+ skip(); await flush();
+ expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); expect(toggle()).not.toBeChecked();
+ expect(f.settings.workflowSharing).toBeNull(); expect(f.settings.piiBackend).toBe('local'); expect(f.settings.disabled).toBe(true);
+ expect(f.settings.workflowSharingPromptSeen).toEqual({ 'synthetic-a': notice }); expect(grants()).toEqual([]);
+});
+it('offers sharing without recorder readiness or automatic-update access', async () => {
+ const tasks = service(); render(<WorkflowTasksPrompt active backendReady={false} canEnable={false} tasks={tasks} />);
+ expect(await dialog()).toBeVisible(); expect(tasks.load).not.toHaveBeenCalled(); expect(tasks.enable).not.toHaveBeenCalled(); expect(grants()).toEqual([]);
+ skip(); await flush(); expect(f.settings.workflowSharing).toBeNull();
+});
+it('records only the skipping account and does not repeat on workspace re-entry', async () => {
+ f.settings.workflowSharingPromptSeen = { other: notice }; const tasks = service();
+ const view = render(<WorkflowTasksPrompt active tasks={tasks} />); await dialog(); skip(); await flush();
+ expect(f.settings.workflowSharingPromptSeen).toEqual({ other: notice, 'synthetic-a': notice });
+ view.rerender(<WorkflowTasksPrompt active={false} tasks={tasks} />); view.rerender(<WorkflowTasksPrompt active tasks={tasks} />); await flush();
+ expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); expect(grants()).toEqual([]);
+});
+it('hydrates and switches account choices without granting on logout', async () => {
+ f.settings.user = null; const tasks = service(); const view = render(<WorkflowTasksPrompt active tasks={tasks} />); await flush();
+ expect(screen.queryByRole('dialog')).not.toBeInTheDocument();
+ f.settings.user = { id: 'synthetic-a' }; view.rerender(<WorkflowTasksPrompt active tasks={tasks} />); await dialog(); skip(); await flush();
+ f.settings.user = { id: 'synthetic-b' }; view.rerender(<WorkflowTasksPrompt active tasks={tasks} />); expect(await dialog()).toBeVisible();
+ expect(f.settings.workflowSharingPromptSeen).toEqual({ 'synthetic-a': notice });
+ f.settings.user = null; view.rerender(<WorkflowTasksPrompt active tasks={tasks} />); await flush();
+ expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); expect(grants()).toEqual([]);
+});
+it('keeps an unanswered choice open through recorder disconnect and reconnect', async () => {
+ const tasks = service(); const view = render(<WorkflowTasksPrompt active tasks={tasks} />); await dialog();
+ view.rerender(<WorkflowTasksPrompt active backendReady={false} tasks={tasks} />); expect(await dialog()).toBeVisible();
+ view.rerender(<WorkflowTasksPrompt active backendReady tasks={tasks} />); expect(await dialog()).toBeVisible();
+ expect(f.update).not.toHaveBeenCalled(); expect(grants()).toEqual([]);
+});
+it('preserves inactive and already-seen account behavior', async () => {
+ const tasks = service(); const view = render(<WorkflowTasksPrompt active={false} tasks={tasks} />); await flush();
+ expect(tasks.load).not.toHaveBeenCalled(); expect(f.request).not.toHaveBeenCalled(); expect(screen.queryByRole('dialog')).not.toBeInTheDocument();
+ f.settings.workflowSharingPromptSeen = { 'synthetic-a': notice }; view.rerender(<WorkflowTasksPrompt active tasks={tasks} />); await flush();
+ expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); expect(grants()).toEqual([]); expect(tasks.enable).not.toHaveBeenCalled();
+});
+it('preserves explicit successful sharing through actual controls', async () => {
+ const tasks = service(true); render(<WorkflowTasksPrompt active tasks={tasks} />); await dialog();
+ await waitFor(() => expect(screen.getByRole('button', { name: 'Allow sharing' })).toBeEnabled());
+ expect(grants()).toEqual([]); fireEvent.click(screen.getByRole('button', { name: 'Allow sharing' }));
+ await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument());
+ expect(grants()).toHaveLength(1); expect(grants()[0][0]).toBe('PUT');
+ expect(grants()[0][1]).toMatchObject({ sharing: true, training: true, noticeVersion: notice });
+ expect(f.settings.workflowSharing).toMatchObject({ accountId: 'synthetic-a', epoch: 'synthetic-epoch', priorBackend: 'local' });
+ expect(f.settings.piiBackend).toBe('tinfoil'); expect(f.settings.disabled).toBe(true); expect(tasks.enable).not.toHaveBeenCalled();
+});
+it('preserves unavailable sharing refusal and a usable skip', async () => {
+ f.request.mockImplementation(async () => status({ available: false }));
+ render(<WorkflowTasksPrompt active tasks={service(true)} />); await dialog(); await flush();
+ expect(screen.getByRole('button', { name: 'Allow sharing' })).toBeDisabled(); skip(); await flush();
+ expect(grants()).toEqual([]); expect(f.settings.workflowSharing).toBeNull(); expect(screen.queryByRole('dialog')).not.toBeInTheDocument();
+});
+it('preserves schedule confirmation after an independent skip', async () => {
+ const tasks = service(); render(<WorkflowTasksPrompt active tasks={tasks} />); await dialog(); skip(); await flush();
+ await waitFor(() => expect(toggle()).toBeEnabled()); fireEvent.click(toggle());
+ fireEvent.click(await screen.findByRole('button', { name: 'Enable automatic updates' }));
+ await waitFor(() => expect(toggle()).toBeChecked()); expect(tasks.enable).toHaveBeenCalledOnce();
+ expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); expect(grants()).toEqual([]); expect(f.settings.workflowSharing).toBeNull();
+});