YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover manual sync HTTP failure feedback
Louis Beaumont0a8c65425465
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 89916863d..b46e64bd7 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.This does not establish native recording continuity, real authentication,account-switch race safety, browser integration, isolation or model performance.-The current app corpus contains 127 git-mined regressions. See+The current app corpus contains 128 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1572,3 +1572,16 @@ language shortcuts, lost audio and missing-source/syntax infrastructure errors.Fixtures appear only at grading and need no dependency links. This does notestablish speech recognition quality, native settings persistence, real providerdelivery, alternate-provider behavior, execution isolation or model improvement.++## Manual sync HTTP failures++`app-manual-sync-http-failure` mounts AccountSection with synthetic local HTTP,+settings and native ports. Sixteen outcomes cover pull/push refusal, server-error+feedback, non-JSON status fallback, retry, successful sequencing, network errors+and disabled choices. The parent fails nine outcomes and preserves seven; the+historical reference and current component pass sixteen. Run+`bun test evals/coding-agent/calibrate-manual-sync.test.js` for correct, broken,+bypass, equivalent, preserved-behavior and setup-error controls. Hidden fixtures+and dependency links appear only after the trajectory. This does not establish+native persistence, encryption, real account access, provider delivery, execution+isolation or model improvement.diff --git a/evals/coding-agent/calibrate-manual-sync.test.js b/evals/coding-agent/calibrate-manual-sync.test.jsnew file mode 100644index 000000000..a88ecf60a--- /dev/null+++ b/evals/coding-agent/calibrate-manual-sync.test.js@@ -0,0 +1,47 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-manual-sync-http-failure');+const options = `${app}/lib/sync-fetch.ts`, section = `${app}/components/settings/account-section.tsx`;+const root = mkdtempSync(join(tmpdir(), 'manual-sync-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);++ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('16 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent reproduces nine assertions and preserves seven outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('9 failed | 7 passed');},60000);+test('reference passes all outcomes',()=>passes(grade('reference')),60000);+test('current source passes all outcomes',()=>passes(grade('current','HEAD')),60000);+test('unused correct helper cannot repair the caller',()=>{const r=grade('unused',item.oracle_ref,cwd=>writeFileSync(join(cwd,section),execFileSync('git',['show',`${item.base_ref}:${section}`],{cwd:repo})));fails(r);expect(r.stdout).toContain('9 failed | 7 passed');},60000);+test('equivalent helper name and no test ids pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{for(const file of [options,section]){const path=join(cwd,file);writeFileSync(path,readFileSync(path,'utf8').replaceAll('syncFetchOrThrow','requestSync').replaceAll('data-testid=','data-neutral='));}})),60000);+test('blanket rejection loses preserved success',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,options,'if (response.ok) return response;','if (response.ok) throw new Error("blanket rejected");'))),60000);+test('skipping push cannot claim sync success',()=>fails(grade('skip-push',item.oracle_ref,cwd=>{const f=join(cwd,section);const t=readFileSync(f,'utf8');expect(t.match(/await syncFetchOrThrow\("\/sync\/(pipes|memories|connections)\/push", \{ method: "POST" \}\);/g)).toHaveLength(3);writeFileSync(f,t.replace(/await syncFetchOrThrow\("\/sync\/(pipes|memories|connections)\/push", \{ method: "POST" \}\);/g,'void 0;'));})),60000);+test('missing source is infrastructure not baseline behavior',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,section)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);+test('equivalent inline HTTP validation passes',()=>passes(grade('inline',item.oracle_ref,cwd=>{const f=join(cwd,section);const helper=readFileSync(join(cwd,options),'utf8').replace('import { localFetch } from "@/lib/api";','').replace('export async function','async function');replace(cwd,section,'import { syncFetchOrThrow } from "@/lib/sync-fetch";','import { localFetch } from "@/lib/api";');writeFileSync(f,readFileSync(f,'utf8')+'\n'+helper);})),60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex d1a87f36a..be78126b3 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5237,6 +5237,53 @@],"command": "bun --no-env-file test packages/ai-gateway/eval-transcription-language.test.ts"}+ },+ {+ "id": "app-manual-sync-http-failure",+ "name": "Report manual sync HTTP failures without false success",+ "source": {+ "kind": "git_regression",+ "fix_commit": "ac201302bffe38e978efe574fc7033436ae96939"+ },+ "base_ref": "b2ff035b626e7a4a5e117fb1013aa1501d689300",+ "oracle_ref": "ac201302bffe38e978efe574fc7033436ae96939",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/settings/account-section.tsx",+ "apps/screenpipe-app-tauri/lib/sync-fetch.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/settings/account-section.tsx",+ "apps/screenpipe-app-tauri/lib/sync-fetch.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "sync",+ "error-feedback"+ ],+ "prompt": "The account settings manual sync actions can report success when the local server rejects a request. For each enabled pipe/scheduled-task, memory and connection sync action, POST pull before push. A failed pull must stop before push; a failed push must report failure after the successful pull. On HTTP errors, show one destructive failure notification containing the nonempty server error message, or the HTTP status when the body is not JSON. Do not show success on rejection. After a failure, restore the enabled manual action and allow a later successful retry. Preserve one success notification after successful pull and push, with a title indicating synced. Preserve network-error feedback explaining that the server is unreachable and restore the action afterward. Disabled sync choices must expose no manual action and issue no sync request just from opening settings. Keep the existing public AccountSection interface, recognizable sync section labels and Sync now actions. Do not require any particular helper, layout, DOM order or exact notification title. Tests mount the actual component with synthetic account settings, local HTTP responses and inert native/translation ports. This task does not establish native sync persistence, encryption, real account access or backend delivery.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/manual-sync.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-manual-sync.test.tsx"+ },+ {+ "local_path": "graders/manual-sync.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-manual-sync.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-manual-sync.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/manual-sync.config.mjs b/evals/coding-agent/graders/manual-sync.config.mjsnew file mode 100644index 000000000..b64eeda98--- /dev/null+++ b/evals/coding-agent/graders/manual-sync.config.mjs@@ -0,0 +1,13 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import path from 'node:path';+export default defineConfig({+ esbuild: { jsx: 'automatic' },+ plugins: [{name: 'synthetic-translation-port', resolveId(id) { if(id === 'gt-react') return '\0eval-gt-react'; }, load(id) { if(id === '\0eval-gt-react') return 'export const useGT=()=>t=>t; export const msg=t=>t;'; }}],+ test: { include: ['eval-manual-sync.test.tsx'], environment: 'jsdom',+ setupFiles: ['@testing-library/jest-dom/vitest'],+ pool: 'forks', poolOptions: { forks: { singleFork: true } } },+ resolve: { alias: { '@': path.resolve('.') } },+ css: { postcss: { plugins: [] } },+});diff --git a/evals/coding-agent/graders/manual-sync.fixture.tsx.txt b/evals/coding-agent/graders/manual-sync.fixture.tsx.txtnew file mode 100644index 000000000..a0374938e--- /dev/null+++ b/evals/coding-agent/graders/manual-sync.fixture.tsx.txt@@ -0,0 +1,146 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterEach, describe, expect, it, vi } from "vitest";+import {+ act,+ cleanup,+ fireEvent,+ render,+ screen,+ waitFor,+ within,+} from "@testing-library/react";++const mocks = vi.hoisted(() => ({+ state: { user: {id: "synthetic-user", email: "[email protected]", token: "synthetic-token", cloud_subscribed: true, subscription_plan: "pro"} as any, enabled: true },+ fetch: vi.fn(), toast: vi.fn(),+ updateSettings: vi.fn().mockResolvedValue(undefined),+ loadUser: vi.fn().mockResolvedValue(undefined),+ openLoginWindow: vi.fn().mockResolvedValue(undefined),+ piUpdateConfig: vi.fn().mockResolvedValue(undefined),+ capture: vi.fn(),+ openUrl: vi.fn().mockResolvedValue(undefined),+ eventHandlers: new Map<string, (event: any) => unknown>(),+ listen: vi.fn(async (event: string, handler: (event: any) => unknown) => {+ mocks.eventHandlers.set(event, handler);+ return () => mocks.eventHandlers.delete(event);+ }),+}));++// AccountSection reads everything through useSettings + the tauri `commands`+// object; swap `mocks.state.user` per case. Keep `@/lib/app-entitlement` REAL —+// `isSignedInCloudSubscriber` is the gate under test.+vi.mock("@/lib/hooks/use-settings", () => ({+ useSettings: () => ({+ settings: {+ user: mocks.state.user,+ pipeSyncEnabled: mocks.state.enabled,+ memoriesSyncEnabled: mocks.state.enabled,+ connectionsSyncEnabled: mocks.state.enabled,+ },+ updateSettings: mocks.updateSettings,+ loadUser: mocks.loadUser,+ }),+}));++vi.mock("@/lib/hooks/use-health-check", () => ({+ useHealthCheck: () => ({ isServerDown: false }),+}));++vi.mock("@/lib/hooks/use-managed-policy", () => ({+ useManagedPolicy: () => ({ isManagedDeployment: false }),+}));++vi.mock("@/lib/utils/tauri", () => ({+ commands: {+ openLoginWindow: mocks.openLoginWindow,+ piUpdateConfig: mocks.piUpdateConfig,+ },+}));++vi.mock("@/components/ui/use-toast", () => ({ toast: mocks.toast }));+vi.mock("@/lib/api", () => ({ localFetch: mocks.fetch }));+vi.mock("posthog-js", () => ({ default: { capture: mocks.capture } }));++// Tauri plugins the effect wires up on mount — keep them inert.+vi.mock("@tauri-apps/plugin-shell", () => ({ open: mocks.openUrl }));+vi.mock("@tauri-apps/plugin-deep-link", () => ({+ onOpenUrl: vi.fn().mockResolvedValue(() => {}),+}));+vi.mock("@tauri-apps/api/event", () => ({+ listen: mocks.listen,+}));++// ReferralCard pulls its own data deps; it is irrelevant to the gate.+vi.mock("@/components/settings/referral-card", () => ({+ ReferralCard: () => null,+}));+vi.mock("@/components/settings/business-upgrade-card", () => ({+ BusinessUpgradeCard: ({+ onContinue,+ }: {+ onContinue: (selection: any) => void;+ }) => (+ <button+ type="button"+ onClick={() =>+ onContinue({+ interval: "month",+ offerVersion: "business-desktop-v1",+ experimentKey: "desktop-business-upgrade-offer",+ experimentVariant: "control",+ source: "app-account-section",+ })+ }+ >+ upgrade to business+ </button>+ ),+}));++import { AccountSection } from "@/components/settings/account-section";+++vi.mock("gt-react", () => ({useGT: () => (t: string) => t, msg: (t: string) => t}));+vi.mock("@/lib/i18n/provider", () => ({useUiLocale: () => "en"}));+vi.mock("@/lib/http/tauri-fetch", () => ({tauriFetchWithDeadline: async () => new Response("{}")}));+vi.mock("@tauri-apps/plugin-os", () => ({hostname: async () => "synthetic-device"}));+vi.mock("@/components/settings/account-data-sync-card", () => ({AccountDataSyncCard: () => null}));+vi.mock("@/components/settings/sync-key-recovery", () => ({isLegacySyncKeyMismatch: () => false, SyncKeyRecovery: () => null}));+afterEach(() => {cleanup(); vi.clearAllMocks(); vi.unstubAllGlobals(); mocks.state.enabled=true; mocks.fetch.mockReset();});+function mount() {vi.stubGlobal("fetch",vi.fn(async () => {throw Error("unexpected external request")})); return render(<AccountSection/>);}+function button(index: number) {+ const labels=[/^(pipe sync|sync scheduled tasks) across devices$/i,/^memories sync across devices$/i,/^connection sync across devices$/i];+ const label=screen.getByText(labels[index]);let group=label.parentElement;+ while(group && !within(group).queryByRole("button",{name:/^sync now$/i}))group=group.parentElement;+ expect(group).not.toBeNull();return within(group!).getByRole("button",{name:/^sync now$/i});+}+const targets=["pipes","memories","connections"];+function reply(status:number, body:any) {return new Response(JSON.stringify(body),{status,headers:{"Content-Type":"application/json"}});}+for (const [index,target] of targets.entries()) {+ it(target+" pull rejection prevents push and success",async()=>{+ mocks.fetch.mockResolvedValue(reply(400,{error:"synthetic sync refused"}));mount();fireEvent.click(button(index));+ await waitFor(()=>expect(mocks.toast).toHaveBeenCalledTimes(1));+ expect(mocks.toast.mock.calls[0][0]).toMatchObject({variant:"destructive",description:expect.stringContaining("synthetic sync refused")});+ expect(mocks.fetch.mock.calls.map(x=>x[0])).toEqual(["/sync/"+target+"/pull"]);expect(button(index)).not.toBeDisabled();+ });+ it(target+" push rejection is failure after successful pull",async()=>{+ mocks.fetch.mockResolvedValueOnce(reply(200,{})).mockResolvedValueOnce(reply(401,{error:"synthetic unauthorized"}));mount();fireEvent.click(button(index));+ await waitFor(()=>expect(mocks.toast).toHaveBeenCalledTimes(1));expect(mocks.toast.mock.calls[0][0]).toMatchObject({variant:"destructive",description:expect.stringContaining("synthetic unauthorized")});+ expect(mocks.fetch.mock.calls.map(x=>x[0])).toEqual(["/sync/"+target+"/pull","/sync/"+target+"/push"]);expect(button(index)).not.toBeDisabled();+ });+ it(target+" preserves successful pull then push",async()=>{+ mocks.fetch.mockResolvedValue(reply(200,{}));mount();fireEvent.click(button(index));+ await waitFor(()=>expect(mocks.toast).toHaveBeenCalledTimes(1));expect(mocks.toast.mock.calls[0][0].variant).not.toBe("destructive");expect(mocks.toast.mock.calls[0][0].title).toMatch(/synced/i);+ expect(mocks.fetch.mock.calls.map(x=>[x[0],x[1].method])).toEqual([["/sync/"+target+"/pull","POST"],["/sync/"+target+"/push","POST"]]);expect(button(index)).not.toBeDisabled();+ });+ it(target+" handles non JSON response and retries",async()=>{+ mocks.fetch.mockResolvedValueOnce(new Response("synthetic unavailable",{status:503}));mount();fireEvent.click(button(index));+ await waitFor(()=>expect(mocks.toast).toHaveBeenCalledTimes(1));expect(mocks.toast.mock.calls[0][0]).toMatchObject({variant:"destructive",description:expect.stringContaining("503")});+ mocks.fetch.mockResolvedValue(reply(200,{}));fireEvent.click(button(index));await waitFor(()=>expect(mocks.toast).toHaveBeenCalledTimes(2));expect(mocks.toast.mock.calls[1][0].title).toMatch(/synced/i);expect(mocks.fetch).toHaveBeenCalledTimes(3);+ });+ it(target+" preserves network failure feedback",async()=>{+ mocks.fetch.mockRejectedValue(new TypeError("Load failed"));mount();fireEvent.click(button(index));await waitFor(()=>expect(mocks.toast).toHaveBeenCalledTimes(1));expect(mocks.toast.mock.calls[0][0]).toMatchObject({variant:"destructive"});expect(String(mocks.toast.mock.calls[0][0].description)).toMatch(/server.*reachable/i);expect(mocks.fetch).toHaveBeenCalledTimes(1);expect(button(index)).not.toBeDisabled();+ });+}+it("disabled sync choices do not expose manual actions or send requests",()=>{mocks.state.enabled=false;mount();expect(screen.queryAllByRole("button",{name:/^sync now$/i})).toHaveLength(0);expect(mocks.fetch).not.toHaveBeenCalled();});