YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover stalled enterprise policy verification
Louis Beaumont958e8b96bb58
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 0c1fb3d96..44214bbec 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1467,3 +1467,9 @@ continuity, old-row backfill, whitespace-split entities, live model providers,agent isolation or model capability. Run the shared runner with`--case app-ocr-derived-copy-retry --verify`; compilation and extraction failuresmust remain infrastructure errors, not evidence of the historical defect.++### Enterprise local policy waits++`app-enterprise-local-policy-waits` runs the actual enterprise authentication hook with synthetic HTTP, settings and native-command boundaries. Fourteen outcomes cover six stalled local operations, late install metadata, rejected/expired credentials, credential choice, account-only access, recording pause, native denial and healthy policy persistence. The task allows ten seconds for one stalled operation; it does not require a particular timer helper or warning message.++Run `bun test evals/coding-agent/calibrate-enterprise-policy-waits.test.js` to check the broken parent, historical reference, current hook, equivalent helper names, disconnected correct source, omitted persistence, excessive wait, native/pause bypasses and missing-source infrastructure failure. These checks do not establish native recording enforcement, real disk durability, host isolation or model performance.diff --git a/evals/coding-agent/calibrate-enterprise-policy-waits.test.js b/evals/coding-agent/calibrate-enterprise-policy-waits.test.jsnew file mode 100644index 000000000..3840280a0--- /dev/null+++ b/evals/coding-agent/calibrate-enterprise-policy-waits.test.js@@ -0,0 +1,53 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-enterprise-local-policy-waits');+const hook = `${app}/lib/hooks/use-enterprise-policy.ts`;+const root = mkdtempSync(join(tmpdir(), 'enterprise-policy-waits-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.includes(from)).toBe(true);+ writeFileSync(p, text.replaceAll(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('14 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('broken parent exposes stalled authentication and preserves healthy behavior',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('9 failed | 5 passed');},60000);+test('historical reference passes all outcomes',()=>passes(grade('reference')),60000);+test('current hook passes all outcomes',()=>passes(grade('current','HEAD')),60000);+test('equivalent helper names pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ const file=join(cwd,hook);writeFileSync(file,readFileSync(file,'utf8').replaceAll('readPolicySettings','loadLocalPolicyState').replaceAll('savePolicySettings','persistLocalPolicyState'));+})),60000);+test('unused correct copy cannot hide broken active caller',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/unused-correct-policy.ts`),readFileSync(join(cwd,hook)));+ writeFileSync(join(cwd,hook),execFileSync('git',['show',`${item.base_ref}:${hook}`],{cwd:repo}));+})),60000);+test('removing all policy persistence fails healthy preservation',()=>fails(grade('no-persistence',item.oracle_ref,cwd=>replace(cwd,hook,' const { store, settings } = await readPolicySettings();\n const metadata = await getEnterpriseInstallMetadata();',' return {install_source:"unknown", update_manager:"unknown", managed:false, detected_by:[]};\n const { store, settings } = await readPolicySettings();\n const metadata = await getEnterpriseInstallMetadata();'))),60000);+test('a deadline beyond the task budget fails',()=>fails(grade('late-deadline',item.oracle_ref,cwd=>replace(cwd,hook,'const LOCAL_POLICY_COMMAND_TIMEOUT_MS = 8_000;','const LOCAL_POLICY_COMMAND_TIMEOUT_MS = 30_000;'))),60000);+test('missing caller is infrastructure failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');},60000);+test('bypassing native grant failure is rejected',()=>fails(grade('native-bypass',item.oracle_ref,cwd=>replace(cwd,hook,'if (!await setNativeRecordingAuthorized(true, credential)) {','if (false && !await setNativeRecordingAuthorized(true, credential)) {'))),60000);+test('ignoring administrator pause is rejected',()=>fails(grade('pause-bypass',item.oracle_ref,cwd=>replace(cwd,hook,'if (!result.policy.recordingAllowed) {','if (false && !result.policy.recordingAllowed) {'))),60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 6bf542b9f..fafd079a1 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -4886,6 +4886,50 @@],"command": "cargo test --locked -p screenpipe-redact --test eval_ocr_derived -- --test-threads=1"}+ },+ {+ "id": "app-enterprise-local-policy-waits",+ "name": "Finish enterprise verification despite stalled local policy work",+ "source": {+ "kind": "git_regression",+ "fix_commit": "6b48cdbc6bd445f6ce0389aa00fd40c642872f65",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/7435"+ },+ "base_ref": "959f7e46559620b9df4666553379d26d21da8760",+ "oracle_ref": "6b48cdbc6bd445f6ce0389aa00fd40c642872f65",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-enterprise-policy.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-enterprise-policy.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "enterprise",+ "auth",+ "deadline",+ "preserved-behavior"+ ],+ "prompt": "Enterprise startup can stay on checking access forever when local policy work stalls, even though the service can verify the saved license key. Ensure verification finishes within 10 seconds when one store-open, settings-read, AI-preset write/save, install-metadata lookup, or app-update settings-save operation never settles and the other operations respond promptly. Continue real policy verification and native authorization before reporting success. Keep key requests free of account authorization headers. Ignore install metadata that arrives after its wait has ended so it cannot overwrite settings or a newly selected account. Preserve credential choice with no credentials, denied and expired credentials even after a local read stall, account-login requirements, administrator recording pauses, and native authorization failure. Healthy policy application must still persist update policy and timely install metadata while retaining allowed employee presets, account identity, and unrelated settings.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/enterprise-policy-waits.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/lib/hooks/__tests__/eval-enterprise-policy-waits.test.tsx"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts lib/hooks/__tests__/eval-enterprise-policy-waits.test.tsx"+ }}]}diff --git a/evals/coding-agent/graders/enterprise-policy-waits.fixture.tsx.txt b/evals/coding-agent/graders/enterprise-policy-waits.fixture.tsx.txtnew file mode 100644index 000000000..8e6e11cb0--- /dev/null+++ b/evals/coding-agent/graders/enterprise-policy-waits.fixture.tsx.txt@@ -0,0 +1,271 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { act, cleanup, renderHook } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => {+ const webBaseOverride = { value: null as string | null };+ const settings: Record<string, unknown> = { deviceId: "device-1" };+ const store = {+ get: vi.fn(async (key: string) => (key === "settings" ? settings : undefined)),+ set: vi.fn(async (key: string, value: unknown) => {+ if (key === "settings") {+ Object.keys(settings).forEach((k) => delete settings[k]);+ Object.assign(settings, value as Record<string, unknown>);+ }+ }),+ save: vi.fn(async () => undefined),+ };++ return {+ webBaseOverride,+ settings,+ store,+ isEnterprise: { value: true },+ commands: {+ getEnterpriseLicenseKey: vi.fn(async () => null as string | null),+ saveEnterpriseLicenseKey: vi.fn(async () => ({ status: "ok" as const })),+ getEnterpriseInstallMetadata: vi.fn(async () => ({+ install_source: "unknown",+ update_manager: "unknown",+ managed: false,+ detected_by: [],+ })),+ getEnterpriseHostIdentity: vi.fn(async () => ({+ machine_id_hash: "sp_machine_v1_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ os_user_id_hash: "sp_os_user_v1_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ })),+ getCloudToken: vi.fn(async () => null as string | null),+ stopScreenpipe: vi.fn(async () => undefined),+ spawnScreenpipe: vi.fn(async () => undefined),+ setEnterprisePolicy: vi.fn(async () => undefined),+ setEnterpriseRecordingAuthorized: vi.fn(async () => ({ status: "ok", data: null })),+ applyEnterpriseUiVisibility: vi.fn(async () => undefined),+ setSyncStreams: vi.fn(async () => undefined),+ setActivitySyncEnabled: vi.fn(async () => undefined),+ saveEnterpriseTeamConfig: vi.fn(async () => null),+ syncManagedTeamSkills: vi.fn(async () => ({ status: "ok" as const, data: [] })),+ },+ tauriFetch: vi.fn(),+ localFetch: vi.fn(),+ getVersion: vi.fn(async () => "2.5.99"),+ platform: vi.fn(() => "windows"),+ syncManagedPipes: vi.fn(async () => undefined),+ gatherPipeStatuses: vi.fn(async () => []),+ };+});++vi.mock("@/lib/hooks/use-is-enterprise-build", () => ({+ useEnterpriseBuildStatus: () => ({+ isEnterprise: mocks.isEnterprise.value,+ resolved: true,+ error: false,+ }),+}));++vi.mock("@/lib/hooks/use-settings", () => ({+ getStore: vi.fn(async () => mocks.store),+ useSettings: () => ({ settings: mocks.settings }),+}));++vi.mock("@/lib/utils/tauri", () => ({+ commands: mocks.commands,+}));++vi.mock("@tauri-apps/plugin-http", () => ({+ fetch: mocks.tauriFetch,+}));++// The baked control-plane base is read at module load from a NEXT_PUBLIC env+// var, so it has to be mocked to be varied per test. Both the hook and the+// shared fetch wrapper import this module, so overriding it here exercises the+// real chain: web base -> isLocalControlPlaneBase() -> which deadline applies.+vi.mock("@/lib/web-url", () => ({+ PROD_WEB_BASE: "https://screenpipe.com",+ screenpipeWebBase: (fallbackHost: string) => mocks.webBaseOverride.value ?? fallbackHost,+ screenpipeWebUrl: (path: string, fallbackHost: string) =>+ `${mocks.webBaseOverride.value ?? fallbackHost}${path.startsWith("/") ? path : `/${path}`}`,+}));++vi.mock("@tauri-apps/api/app", () => ({+ getVersion: mocks.getVersion,+}));++vi.mock("@tauri-apps/plugin-os", () => ({+ platform: mocks.platform,+}));++vi.mock("@/lib/api", () => ({+ localFetch: mocks.localFetch,+}));++vi.mock("@/lib/hooks/use-enterprise-pipes", () => ({+ syncManagedPipes: mocks.syncManagedPipes,+ gatherPipeStatuses: mocks.gatherPipeStatuses,+}));++import { useEnterprisePolicyRuntime } from "@/lib/hooks/use-enterprise-policy";+import { getStore } from "@/lib/hooks/use-settings";+++const KEY = "ENT-TEST-ONLY-HOOK-0001";++function policyResponse(overrides: Record<string, unknown> = {}) {+ return new Response(+ JSON.stringify({+ hiddenSections: [],+ lockedSettings: {},+ managedPipes: [],+ orgName: "Synthetic Harbor",+ ...overrides,+ }),+ { status: 200 }+ );+}++function heartbeatResponse(status = 200, code?: string) {+ return new Response(+ JSON.stringify(status === 200 ? { ok: true } : { error: "x", code }),+ { status },+ );+}++function mockEnterpriseApi(opts: {+ policyStatus?: number;+ policy?: Record<string, unknown>;+ heartbeatStatus?: number;+ heartbeatCode?: string;+}) {+ mocks.tauriFetch.mockImplementation(async (url: string) => {+ if (url.includes("/api/enterprise/policy")) {+ if (opts.policyStatus && opts.policyStatus !== 200) {+ return new Response(JSON.stringify({ error: "bad credential" }), {+ status: opts.policyStatus,+ });+ }+ return policyResponse(opts.policy);+ }+ if (url.includes("/api/enterprise/heartbeat")) {+ return heartbeatResponse(opts.heartbeatStatus ?? 200, opts.heartbeatCode);+ }+ throw new Error(`unexpected fetch ${url}`);+ });+}+++// Test only the public hook. Native, store and HTTP boundaries are synthetic.+const stages = ["store open", "settings read", "AI preset write", "AI preset save", "install metadata", "app update save"];+function stall(stage: string) {+ const pending = new Promise<never>(() => {});+ if (stage === "store open") vi.mocked(getStore).mockReturnValueOnce(pending);+ if (stage === "settings read") mocks.store.get.mockReturnValueOnce(pending);+ if (stage === "AI preset write") mocks.store.set.mockReturnValueOnce(pending);+ if (stage === "AI preset save") mocks.store.save.mockReturnValueOnce(pending);+ if (stage === "install metadata") mocks.commands.getEnterpriseInstallMetadata.mockReturnValueOnce(pending);+ if (stage === "app update save") mocks.store.save.mockResolvedValueOnce(undefined).mockReturnValueOnce(pending);+}+async function start() {+ const hook = renderHook(() => useEnterprisePolicyRuntime());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(10_001); });+ return hook;+}+function noGrant() {+ expect(mocks.commands.setEnterpriseRecordingAuthorized.mock.calls.some(args => args[0] === true)).toBe(false);+}+describe("enterprise local policy wait outcomes", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.useFakeTimers();+ localStorage.clear();+ vi.mocked(getStore).mockReset().mockResolvedValue(mocks.store as any);+ mocks.store.get.mockReset().mockImplementation(async key => key === "settings" ? mocks.settings : undefined);+ mocks.store.set.mockReset().mockImplementation(async (key, value) => {+ if (key === "settings") {+ Object.keys(mocks.settings).forEach(k => delete mocks.settings[k]);+ Object.assign(mocks.settings, value);+ }+ });+ mocks.store.save.mockReset().mockResolvedValue(undefined);+ mocks.commands.getEnterpriseInstallMetadata.mockReset().mockResolvedValue({install_source:"unknown", update_manager:"unknown", managed:false, detected_by:[]});+ mocks.commands.getEnterpriseLicenseKey.mockReset().mockResolvedValue(KEY);+ mocks.commands.setEnterpriseRecordingAuthorized.mockReset().mockResolvedValue({status:"ok", data:null});+ mocks.webBaseOverride.value = null;+ mocks.isEnterprise.value = true;+ Object.keys(mocks.settings).forEach(k => delete mocks.settings[k]);+ Object.assign(mocks.settings, {deviceId:"device-1"});+ mocks.localFetch.mockResolvedValue(new Response(JSON.stringify({frame_status:"ok",audio_status:"ok",hostname:"synthetic-host"})));+ mockEnterpriseApi({});+ });+ afterEach(() => { cleanup(); vi.useRealTimers(); });++ it.each(stages)("finishes a valid saved-key verification with stalled %s", async stage => {+ stall(stage);+ const {result} = await start();+ expect(result.current.authenticationState).toBe("authenticated");+ expect(result.current.isEnterpriseAuthenticated).toBe(true);+ expect(mocks.commands.setEnterpriseRecordingAuthorized).toHaveBeenCalledWith(true,"license_key",KEY);+ const call = mocks.tauriFetch.mock.calls.find(([url]) => String(url).includes("/api/enterprise/policy"));+ expect(call?.[1]?.headers["X-License-Key"]).toBe(KEY);+ expect(call?.[1]?.headers.Authorization).toBeUndefined();+ });++ it("ignores metadata delivered after its wait ended without overwriting a newer account", async () => {+ let resolve!: (value:any) => void;+ mocks.commands.getEnterpriseInstallMetadata.mockReturnValueOnce(new Promise(r => {resolve=r;}));+ const {result} = await start();+ expect(result.current.isEnterpriseAuthenticated).toBe(true);+ mocks.settings.user = {token:"synthetic-new-account"};+ const before = JSON.stringify(mocks.settings);+ await act(async () => { resolve({install_source:"mdm",update_manager:"mdm",managed:true,detected_by:[]}); });+ expect(JSON.stringify(mocks.settings)).toBe(before);+ });++ it.each([401,402])("keeps HTTP %s credentials unauthorized despite a stalled settings read", async status => {+ stall("settings read"); mockEnterpriseApi({policyStatus:status});+ const {result} = await start();+ expect(result.current.authenticationState).not.toBe("checking");+ expect(result.current.isEnterpriseAuthenticated).toBe(false);+ noGrant();+ });+ it("offers a credential choice without credentials", async () => {+ mocks.commands.getEnterpriseLicenseKey.mockResolvedValue(null);+ const {result} = await start();+ expect(result.current.authenticationState).toBe("choice");+ expect(result.current.isEnterpriseAuthenticated).toBe(false);+ expect(mocks.tauriFetch).not.toHaveBeenCalled(); noGrant();+ });+ it("preserves an administrator's recording pause", async () => {+ mockEnterpriseApi({policy:{recordingAllowed:false},heartbeatStatus:403,heartbeatCode:"recording_disabled_by_admin"});+ const {result} = await start();+ expect(result.current.isEnterpriseAuthenticated).toBe(true);+ expect(result.current.policy.recordingAllowed).toBe(false); noGrant();+ });+ it("requires account sign-in when key-only access is prohibited", async () => {+ mockEnterpriseApi({policy:{requireAccountLogin:true}});+ const {result} = await start();+ expect(result.current.authenticationState).toBe("account");+ expect(result.current.isEnterpriseAuthenticated).toBe(false); noGrant();+ });+ it("does not declare success when native authorization fails", async () => {+ mocks.commands.setEnterpriseRecordingAuthorized.mockResolvedValue({status:"error",error:"synthetic-denial"} as any);+ const {result} = await start();+ expect(result.current.isEnterpriseAuthenticated).toBe(false);+ expect(result.current.authenticationState).not.toBe("checking");+ });+ it("preserves healthy policy application, employee settings and authentication", async () => {+ const preset = {id:"employee-local",provider:"custom",model:"synthetic-model",url:"http://localhost:9999",defaultPreset:true};+ Object.assign(mocks.settings,{user:{token:"synthetic-account"},autoUpdate:true,aiPresets:[preset],unrelated:"keep"});+ mocks.commands.getEnterpriseInstallMetadata.mockResolvedValue({install_source:"mdm",update_manager:"mdm",managed:true,detected_by:[]});+ mockEnterpriseApi({policy:{appUpdatePolicy:{mode:"manual",default_auto_update:false,allow_employee_override:false}}});+ const {result} = await start();+ expect(result.current.isEnterpriseAuthenticated).toBe(true);+ expect(mocks.settings.autoUpdate).toBe(false);+ expect(mocks.settings.enterpriseInstallMetadata).toMatchObject({managed:true,update_manager:"mdm"});+ expect(mocks.settings.aiPresets).toEqual([preset]);+ expect(mocks.settings.user).toEqual({token:"synthetic-account"});+ expect(mocks.settings.unrelated).toBe("keep");+ expect(mocks.store.save).toHaveBeenCalled();+ });+});