Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): respect settings telemetry identity refusal

louis6d819310bcaf
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index af3044ea1..054a3d0d2 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1122,3 +1122,18 @@ Set `EVAL_CALIBRATION_RESULTS_DIR` to retain the per-control logs. No installed
packages are needed, and the hidden fixture is materialized only at grading.
These outcomes do not prove workerd/D1 concurrency or atomicity, deployed cron
delivery, full quota admission, agent isolation or model improvement.
+
+## Settings identity refusal
+
+`app-settings-telemetry-identity` renders the real SettingsProvider with synthetic
+settings, native and PostHog ports. Five outcomes cover explicit user refusal,
+runtime refusal, a pending runtime decision, opted-in account identity and
+unmounting before version completion. The historical parent fails four and
+preserves one; reference and current source pass five.
+
+Run `bun test evals/coding-agent/calibrate-telemetry-identity.test.js` for eight
+controls, including unused correct source, blanket suppression, ignored user
+preference, equivalent private names and missing-source setup classification.
+Fixtures and dependencies are installed only when grading starts. This does not
+establish native environment parsing, actual capture delivery, unknown preference
+or failed IPC policy, execution isolation or model performance.
diff --git a/evals/coding-agent/calibrate-telemetry-identity.test.js b/evals/coding-agent/calibrate-telemetry-identity.test.js
new file mode 100644
index 000000000..b54c1325b
--- /dev/null
+++ b/evals/coding-agent/calibrate-telemetry-identity.test.js
@@ -0,0 +1,51 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-settings-telemetry-identity');
+const hook = `${app}/lib/hooks/use-settings.tsx`;
+const root = mkdtempSync(join(tmpdir(), 'telemetry-identity-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('5 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails four identity outcomes and preserves opted-in identity',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 1 passed');},60000);
+test('historical reference passes five outcomes',()=>passes(grade('reference')),60000);
+test('current caller preserves five outcomes',()=>passes(grade('current','HEAD')),60000);
+test('equivalent private gate names pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{
+ const file=join(cwd,hook);writeFileSync(file,readFileSync(file,'utf8').replaceAll('identifyNow','emitPermittedIdentity').replaceAll('cancelled','effectEnded'));
+})),60000);
+test('unused correct source cannot hide the broken active caller',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ writeFileSync(join(cwd,`${app}/unused-correct-settings.tsx`),readFileSync(join(cwd,hook)));
+ writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}));
+})),60000);
+test('blanket identity suppression fails opted-in preservation',()=>fails(grade('no-identify',item.oracle_ref,cwd=>replace(cwd,hook,'if (!settings.analyticsId) return;','if (true) return;'))),60000);
+test('ignoring the explicit user refusal fails',()=>fails(grade('ignore-preference',item.oracle_ref,cwd=>replace(cwd,hook,'analyticsEnabled: settings.analyticsEnabled, envDisabled','analyticsEnabled: true, envDisabled'))),60000);
+test('missing caller is a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 0454c37ec..fe3cb64d6 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -3888,6 +3888,50 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-stale-session.test.tsx"
}
+ },
+ {
+ "id": "app-settings-telemetry-identity",
+ "name": "Respect runtime and user refusal before settings identity effects",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "d57b2fb92d84f45a2b2374146a7fc0300ae95def"
+ },
+ "base_ref": "0bae3b170b3cd603381df4278d5e6311ba857734",
+ "oracle_ref": "d57b2fb92d84f45a2b2374146a7fc0300ae95def",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx",
+ "apps/screenpipe-app-tauri/lib/telemetry-env.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx",
+ "apps/screenpipe-app-tauri/lib/telemetry-env.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "privacy",
+ "telemetry",
+ "async-lifecycle"
+ ],
+ "prompt": "The settings provider can alias and identify an account even when the saved analytics preference is explicitly disabled or the native runtime reports telemetry disabled. Repair the actual SettingsProvider identity effect. Neither alias nor identify may run for either refusal, or while the native runtime decision is pending. Once that decision permits telemetry and the user explicitly opted in, preserve account aliasing and identification by the Clerk account ID, including the saved email and stable per-install analytics ID. Unmounting the provider while the app-version lookup is pending must prevent a late identify call. Keep the existing public provider/settings interfaces and native command contract compatible. Tests use actual React lifecycle with synthetic native, settings-store and PostHog ports. No particular private helper or gating structure is required. Native environment parsing, capture transport, unknown preferences and failed IPC policy are outside this task.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/telemetry-identity.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-telemetry-caller.test.tsx"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-telemetry-caller.test.tsx"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/telemetry-identity.fixture.tsx.txt b/evals/coding-agent/graders/telemetry-identity.fixture.tsx.txt
new file mode 100644
index 000000000..79cb5a943
--- /dev/null
+++ b/evals/coding-agent/graders/telemetry-identity.fixture.tsx.txt
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import React from 'react';
+import {render,act,waitFor,cleanup} from '@testing-library/react';
+import {it,expect,vi,afterEach,beforeEach} from 'vitest';
+const state=vi.hoisted(()=>({settings:null as any, runtime:Promise.resolve(false),version:Promise.resolve('0.0.0'),identify:vi.fn(),alias:vi.fn(),optOut:vi.fn(),unexpected:[] as string[]}));
+vi.mock('posthog-js',()=>({default:{identify:state.identify,alias:state.alias,opt_out_capturing:state.optOut,capture:vi.fn()}}));
+vi.mock('@tauri-apps/api/core',()=>({invoke:(name:string)=>{if(name==='is_telemetry_disabled_by_env')return state.runtime;state.unexpected.push(name);throw Error('unexpected native command');}}));
+vi.mock('@tauri-apps/api/app',()=>({getVersion:()=>state.version}));
+vi.mock('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic'}));
+vi.mock('@tauri-apps/plugin-os',()=>({platform:()=>'macos',arch:()=>'aarch64'}));
+vi.mock('@tauri-apps/api/event',()=>({emit:vi.fn(async()=>{}),listen:vi.fn(async()=>()=>{})}));
+vi.mock('@tauri-apps/plugin-store',()=>({Store:{load:async()=>({get:async()=>structuredClone(state.settings),set:async(_k:string,v:any)=>{state.settings=structuredClone(v)},save:async()=>{},onKeyChange:async()=>()=>{}})}}));
+vi.mock('@/lib/utils/tauri',()=>({commands:{getScreenpipeBaseDir:async()=>({status:'ok',data:'/synthetic'}),reencryptStore:async()=>{},getCloudToken:async()=>null,setCloudToken:async()=>({status:'ok'}),setBrowserCookieAccessState:async()=>{}}}));
+vi.mock('@/lib/auth-guard',()=>({installAuthInterceptor:()=>()=>{}}));
+vi.mock('@/lib/analytics-id',()=>({cacheAnalyticsId:()=>{},cacheAnalyticsEnabled:()=>{}}));
+vi.mock('@/lib/analytics/settings-change',()=>({captureSettingsChange:()=>{}}));
+vi.mock('@/lib/api',()=>({configureApi:()=>{},refreshApiConfig:async()=>{},localFetch:async()=>{throw Error('unexpected network')}}));
+vi.mock('@/lib/hooks/use-is-enterprise-build',()=>({isResolvedConsumerBuild:async()=>true}));
+const gate=()=>{let resolve!:(x:any)=>void;const promise=new Promise<any>(r=>{resolve=r});return {promise,resolve}};
+beforeEach(()=>{vi.resetModules();vi.clearAllMocks();state.unexpected=[];state.runtime=Promise.resolve(false);state.version=Promise.resolve('0.0.0');localStorage.clear();vi.stubGlobal('fetch',vi.fn(async()=>{throw Error('network forbidden')}));});
+afterEach(()=>{cleanup();vi.unstubAllGlobals();expect(state.unexpected).toEqual([])});
+async function mount(enabled:boolean){
+ const m=await import('@/lib/hooks/use-settings');state.settings={...m.createDefaultSettingsObject(),deviceId:'synthetic-device',analyticsId:'synthetic-install',analyticsEnabled:enabled,user:{id:'synthetic-user',clerk_id:'synthetic-clerk',email:'[email protected]'},_proCloudMigrationDone:true,_presetsSeededForUser:true};
+ let loaded = false;
+ function Observer() { loaded = m.useSettings().isSettingsLoaded; return null; }
+ const view=render(<m.SettingsProvider><Observer /></m.SettingsProvider>);
+ await waitFor(()=>expect(loaded).toBe(true));
+ await act(async()=>{await Promise.resolve();await Promise.resolve()});return view;
+}
+it('explicit analytics refusal blocks both identity effects',async()=>{await mount(false);expect(state.identify).not.toHaveBeenCalled();expect(state.alias).not.toHaveBeenCalled()});
+it('native runtime refusal blocks both identity effects',async()=>{state.runtime=Promise.resolve(true);await mount(true);expect(state.identify).not.toHaveBeenCalled();expect(state.alias).not.toHaveBeenCalled()});
+it('opted-in settled runtime preserves account identity',async()=>{await mount(true);await waitFor(()=>expect(state.identify).toHaveBeenCalledWith('synthetic-clerk',expect.objectContaining({email:'[email protected]',machine_analytics_id:'synthetic-install'})));expect(state.alias).toHaveBeenCalledWith('synthetic-clerk')});
+it('pending runtime delays identity until permission resolves',async()=>{const g=gate();state.runtime=g.promise;await mount(true);expect(state.identify).not.toHaveBeenCalled();expect(state.alias).not.toHaveBeenCalled();await act(async()=>g.resolve(false));await waitFor(()=>expect(state.identify).toHaveBeenCalled())});
+it('unmount fences late version completion',async()=>{const g=gate();state.version=g.promise;const v=await mount(true);v.unmount();await act(async()=>g.resolve('0.0.0'));expect(state.identify).not.toHaveBeenCalled()});