YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): reject pipe-store empty catalog false passes
louis08dd8a571402
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 6d4da6fab..18fcd9d88 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -13,6 +13,19 @@ The current app corpus contains 80 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.+`app-pipe-store-load-error` renders the actual store component with synthetic+network, cache and native ports. Ten outcomes distinguish payload, HTTP and+network failures from genuine empty catalogs and verify retries to populated+and empty results without stale error UI. The parent fails six error outcomes+and preserves four catalog outcomes; the reference and current component pass+ten. The earlier grader accepted an empty-catalog denial mutant. Run+`bun test evals/coding-agent/calibrate-pipe-store-error.test.js` for nine controls,+including that bypass, equivalent copy/cache strategy, unused correct code,+disabled retry, blanket errors and missing-source setup classification.+Fixtures and dependency links are installed only when grading starts. This+checks rendered behavior with controlled ports, not real cache coherence,+native registry delivery, complete localization, isolation or model capability.+`app-consumer-stale-managed-settings` runs actual settings read, write and resetoperations, the write queue and build-authority gate against synthetic native,store, React rendering and external ports. Confirmed consumer builds release stalediff --git a/evals/coding-agent/calibrate-pipe-store-error.test.js b/evals/coding-agent/calibrate-pipe-store-error.test.jsnew file mode 100644index 000000000..98955b7c9--- /dev/null+++ b/evals/coding-agent/calibrate-pipe-store-error.test.js@@ -0,0 +1,52 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-pipe-store-load-error');+const component = `${app}/components/pipe-store.tsx`;+const root = mkdtempSync(join(tmpdir(), 'pipe-store-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);++ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('10 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails six error outcomes and preserves four catalogs',()=>{ const r=grade('parent',item.base_ref); fails(r); expect(r.stdout).toContain('6 failed | 4 passed'); },60000);+test('reference passes ten outcomes',()=>passes(grade('reference')),60000);+test('current source preserves the disclosed contract',()=>passes(grade('current','HEAD')),60000);+test('empty-catalog false-pass mutant is rejected',()=>fails(grade('empty-denial',item.oracle_ref,cwd=>replace(cwd,component,' return list;\n}', ' if (list.length === 0) throw new Error("empty unavailable");\n return list;\n}'))),60000);+test('unused correct component cannot hide broken active code',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/components/unused-store.tsx`),readFileSync(join(cwd,component)));+ writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.base_ref}:${component}`],{cwd:repo}));+})),60000);+test('equivalent copy, helper names and cache invalidation pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ const path=join(cwd,component);writeFileSync(path,readFileSync(path,'utf8').replaceAll('getPipeStoreList','decodeCatalog').replaceAll('data-testid=','data-eval-neutral=').replaceAll('couldn't load scheduled tasks','Unable to load catalog').replaceAll('TRY AGAIN','Retry').replaceAll('No scheduled tasks found','Catalog is empty').replaceAll('apiCache.invalidatePrefix("pipes/store");','apiCache.invalidate("pipes/store");'));+})),60000);+test('blanket error display is rejected',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,component,'loadError && pipes.length === 0','true'))),60000);+test('disabled retry cannot recover',()=>fails(grade('no-retry',item.oracle_ref,cwd=>replace(cwd,component,'apiCache.invalidatePrefix("pipes/store");\n void fetchPipes();','void 0;'))),60000);+test('missing component is a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,component)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex e67878e0c..93528d5b5 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-27.2",+ "dataset_version": "2026-09-27.3","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -348,10 +348,10 @@"name": "Distinguish pipe-store failures from an empty catalog","source": {"kind": "git_regression",- "fix_commit": "8dbf6e47b"+ "fix_commit": "8dbf6e47bac2bd033eaca3d7da649fce585b1c5f"},- "base_ref": "8dbf6e47b^",- "oracle_ref": "8dbf6e47b",+ "base_ref": "e541a77536ccf33cd9668f18aad065a95bd19d42",+ "oracle_ref": "8dbf6e47bac2bd033eaca3d7da649fce585b1c5f","tags": ["regression","escaped-failure",@@ -359,23 +359,27 @@"error-state","retry"],- "prompt": "When the pipe store request fails, the app renders the same empty state as a successfully loaded catalog with no pipes. Surface a clear load-error state with a working retry action, keep genuine empty results distinct, and recover to the normal store after a successful retry without stale error UI.",+ "prompt": "When the pipe store request fails, the app renders the same empty state as a successfully loaded catalog with no pipes. Show a clear load-error state and a working retry for network failures, unsuccessful HTTP responses, and error payloads. Keep successful empty catalogs distinct, including bare arrays and lists wrapped in data or pipes. A successful retry to either populated or empty results must clear stale error and retry UI. Preserve the installed-tasks tab and successful catalog content. Wording and internal cache implementation may vary.","agent_timeout_seconds": 900,- "dependency_links": [- {- "source_path": "apps/screenpipe-app-tauri/node_modules",- "destination_path": "apps/screenpipe-app-tauri/node_modules"- }- ],"grader": {"timeout_seconds": 300,"fixtures": [{- "source_path": "apps/screenpipe-app-tauri/components/__tests__/pipe-store-error-state.test.tsx"+ "local_path": "graders/pipe-store-error.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/components/__tests__/pipe-store-error-state.test.tsx"}],- "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/__tests__/pipe-store-error-state.test.tsx"- }+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/__tests__/pipe-store-error-state.test.tsx --maxWorkers=1 --minWorkers=1"+ },+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/pipe-store.tsx"+ ]},{"id": "app-learning-handoff-always-finishes",diff --git a/evals/coding-agent/graders/pipe-store-error.fixture.tsx.txt b/evals/coding-agent/graders/pipe-store-error.fixture.tsx.txtnew file mode 100644index 000000000..24a13a88c--- /dev/null+++ b/evals/coding-agent/graders/pipe-store-error.fixture.tsx.txt@@ -0,0 +1,132 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import React from "react";+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { PipeStoreView } from "@/components/pipe-store";++const mocks = vi.hoisted(() => ({+ localFetch: vi.fn(),+ cacheSet: vi.fn(),+ invalidatePrefix: vi.fn(),+}));++vi.mock("@/lib/api", () => ({ localFetch: mocks.localFetch }));+vi.mock("@/lib/cache", () => ({+ apiCache: {+ get: vi.fn(() => null),+ getStale: vi.fn(() => null),+ isFresh: vi.fn(() => false),+ set: mocks.cacheSet,+ invalidate: vi.fn(),+ invalidatePrefix: mocks.invalidatePrefix,+ },+}));+vi.mock("@/lib/hooks/use-settings", () => ({+ useSettings: () => ({ settings: { user: null } }),+}));+vi.mock("@/lib/hooks/use-event-listener", () => ({ useEventListener: vi.fn() }));+vi.mock("@/components/ui/use-toast", () => ({+ useToast: () => ({ toast: vi.fn() }),+}));+vi.mock("@/components/settings/pipes-section", () => ({+ PipesSection: () => <div>installed tasks</div>,+}));+vi.mock("@/components/notification-bell", () => ({ NotificationBell: () => null }));+vi.mock("@/components/pipe-store-submission", () => ({+ PipeStoreSubmissionDialog: () => null,+}));+vi.mock("@/lib/stores/feedback-store", () => ({+ useFeedbackStore: () => vi.fn(),+}));+vi.mock("@tauri-apps/plugin-opener", () => ({ openUrl: vi.fn() }));+vi.mock("posthog-js", () => ({ default: { capture: vi.fn() } }));++function jsonResponse(data: unknown, status = 200): Response {+ return {+ ok: status >= 200 && status < 300,+ status,+ json: async () => data,+ } as Response;+}++describe("pipe store error state", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.spyOn(console, "error").mockImplementation(() => {});+ const values = new Map<string, string>();+ vi.stubGlobal("localStorage", {+ getItem: (key: string) => values.get(key) ?? null,+ setItem: (key: string, value: string) => values.set(key, value),+ removeItem: (key: string) => values.delete(key),+ clear: () => values.clear(),+ });+ localStorage.setItem("screenpipe:pipes-welcome-dismissed", "1");+ });++ afterEach(() => {+ cleanup();+ localStorage.clear();+ vi.restoreAllMocks();+ vi.unstubAllGlobals();+ });++ const errorText = /(?:couldn.t|cannot|unable to) load|load.*failed|catalog.*unavailable/i;+ const emptyText = /no (?:scheduled tasks|tasks|pipes|workflows) (?:found|available)|catalog is empty/i;+ const retryName = /try again|retry|reload/i;+ const pipe = { slug: "daily-summary", title: "Daily summary", description: "summarizes the day", category: "productivity" };+ function installResponses(store: () => Promise<Response>) {+ mocks.localFetch.mockImplementation(async (url: string) => {+ if (url === "/pipes") return jsonResponse([{ config: { name: "installed" } }]);+ if (url === "/pipes/store/check-updates") return jsonResponse({ data: [] });+ if (url.startsWith("/pipes/store?")) return store();+ throw new Error(`unexpected request: ${url}`);+ });+ }+ function discover() {+ render(<PipeStoreView />);+ expect(screen.getByText("installed tasks")).toBeTruthy();+ fireEvent.click(screen.getByRole("button", { name: /discover/i }));+ }+ function noError() {+ expect(screen.queryAllByText(errorText)).toHaveLength(0);+ expect(screen.queryByRole("button", { name: retryName })).toBeNull();+ }+ for (const failure of ["payload", "http", "network"]) {+ for (const recovered of ["populated", "empty"]) {+ it(`${failure} failure stays distinct from empty and retries to ${recovered}`, async () => {+ let failed = false;+ installResponses(async () => {+ if (!failed) {+ failed = true;+ if (failure === "network") throw new Error("offline");+ return failure === "http" ? jsonResponse({}, 503) : jsonResponse({ error: "registry unavailable" });+ }+ return jsonResponse({ data: recovered === "empty" ? [] : [pipe] });+ });+ discover();+ await waitFor(() => expect(screen.queryAllByText(errorText).length).toBeGreaterThan(0));+ expect(screen.queryByText(emptyText)).toBeNull();+ fireEvent.click(screen.getByRole("button", { name: retryName }));+ await waitFor(() => expect(screen.getByText(recovered === "empty" ? emptyText : "Daily summary")).toBeTruthy());+ noError();+ });+ }+ }+ for (const payload of [[], { data: [] }, { pipes: [] }]) {+ it(`successful empty catalog stays empty: ${JSON.stringify(payload)}`, async () => {+ installResponses(async () => jsonResponse(payload));+ discover();+ await waitFor(() => expect(screen.getByText(emptyText)).toBeTruthy());+ noError();+ });+ }+ it("successful populated catalog preserves its content", async () => {+ installResponses(async () => jsonResponse({ data: [pipe] }));+ discover();+ await waitFor(() => expect(screen.getByText("Daily summary")).toBeTruthy());+ noError();+ expect(screen.queryByText(emptyText)).toBeNull();+ });+});