Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): reject pipe-store empty catalog false passes

louis08dd8a571402
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 6d4da6fab..18fcd9d88 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,19 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-pipe-store-load-error` renders the actual store component with synthetic
+network, cache and native ports. Ten outcomes distinguish payload, HTTP and
+network failures from genuine empty catalogs and verify retries to populated
+and empty results without stale error UI. The parent fails six error outcomes
+and preserves four catalog outcomes; the reference and current component pass
+ten. The earlier grader accepted an empty-catalog denial mutant. Run
+`bun test evals/coding-agent/calibrate-pipe-store-error.test.js` for nine controls,
+including that bypass, equivalent copy/cache strategy, unused correct code,
+disabled retry, blanket errors and missing-source setup classification.
+Fixtures and dependency links are installed only when grading starts. This
+checks rendered behavior with controlled ports, not real cache coherence,
+native registry delivery, complete localization, isolation or model capability.
+
`app-consumer-stale-managed-settings` runs actual settings read, write and reset
operations, the write queue and build-authority gate against synthetic native,
store, React rendering and external ports. Confirmed consumer builds release stale
diff --git a/evals/coding-agent/calibrate-pipe-store-error.test.js b/evals/coding-agent/calibrate-pipe-store-error.test.js
new file mode 100644
index 000000000..98955b7c9
--- /dev/null
+++ b/evals/coding-agent/calibrate-pipe-store-error.test.js
@@ -0,0 +1,52 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-pipe-store-load-error');
+const component = `${app}/components/pipe-store.tsx`;
+const root = mkdtempSync(join(tmpdir(), 'pipe-store-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('10 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails six error outcomes and preserves four catalogs',()=>{ const r=grade('parent',item.base_ref); fails(r); expect(r.stdout).toContain('6 failed | 4 passed'); },60000);
+test('reference passes ten outcomes',()=>passes(grade('reference')),60000);
+test('current source preserves the disclosed contract',()=>passes(grade('current','HEAD')),60000);
+test('empty-catalog false-pass mutant is rejected',()=>fails(grade('empty-denial',item.oracle_ref,cwd=>replace(cwd,component,' return list;\n}', ' if (list.length === 0) throw new Error("empty unavailable");\n return list;\n}'))),60000);
+test('unused correct component cannot hide broken active code',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ writeFileSync(join(cwd,`${app}/components/unused-store.tsx`),readFileSync(join(cwd,component)));
+ writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.base_ref}:${component}`],{cwd:repo}));
+})),60000);
+test('equivalent copy, helper names and cache invalidation pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{
+ const path=join(cwd,component);writeFileSync(path,readFileSync(path,'utf8').replaceAll('getPipeStoreList','decodeCatalog').replaceAll('data-testid=','data-eval-neutral=').replaceAll('couldn't load scheduled tasks','Unable to load catalog').replaceAll('TRY AGAIN','Retry').replaceAll('No scheduled tasks found','Catalog is empty').replaceAll('apiCache.invalidatePrefix("pipes/store");','apiCache.invalidate("pipes/store");'));
+})),60000);
+test('blanket error display is rejected',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,component,'loadError && pipes.length === 0','true'))),60000);
+test('disabled retry cannot recover',()=>fails(grade('no-retry',item.oracle_ref,cwd=>replace(cwd,component,'apiCache.invalidatePrefix("pipes/store");\n void fetchPipes();','void 0;'))),60000);
+test('missing component is a setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,component)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index e67878e0c..93528d5b5 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-27.2",
+ "dataset_version": "2026-09-27.3",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -348,10 +348,10 @@
"name": "Distinguish pipe-store failures from an empty catalog",
"source": {
"kind": "git_regression",
- "fix_commit": "8dbf6e47b"
+ "fix_commit": "8dbf6e47bac2bd033eaca3d7da649fce585b1c5f"
},
- "base_ref": "8dbf6e47b^",
- "oracle_ref": "8dbf6e47b",
+ "base_ref": "e541a77536ccf33cd9668f18aad065a95bd19d42",
+ "oracle_ref": "8dbf6e47bac2bd033eaca3d7da649fce585b1c5f",
"tags": [
"regression",
"escaped-failure",
@@ -359,23 +359,27 @@
"error-state",
"retry"
],
- "prompt": "When the pipe store request fails, the app renders the same empty state as a successfully loaded catalog with no pipes. Surface a clear load-error state with a working retry action, keep genuine empty results distinct, and recover to the normal store after a successful retry without stale error UI.",
+ "prompt": "When the pipe store request fails, the app renders the same empty state as a successfully loaded catalog with no pipes. Show a clear load-error state and a working retry for network failures, unsuccessful HTTP responses, and error payloads. Keep successful empty catalogs distinct, including bare arrays and lists wrapped in data or pipes. A successful retry to either populated or empty results must clear stale error and retry UI. Preserve the installed-tasks tab and successful catalog content. Wording and internal cache implementation may vary.",
"agent_timeout_seconds": 900,
- "dependency_links": [
- {
- "source_path": "apps/screenpipe-app-tauri/node_modules",
- "destination_path": "apps/screenpipe-app-tauri/node_modules"
- }
- ],
"grader": {
"timeout_seconds": 300,
"fixtures": [
{
- "source_path": "apps/screenpipe-app-tauri/components/__tests__/pipe-store-error-state.test.tsx"
+ "local_path": "graders/pipe-store-error.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/components/__tests__/pipe-store-error-state.test.tsx"
}
],
- "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/__tests__/pipe-store-error-state.test.tsx"
- }
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/__tests__/pipe-store-error-state.test.tsx --maxWorkers=1 --minWorkers=1"
+ },
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/pipe-store.tsx"
+ ]
},
{
"id": "app-learning-handoff-always-finishes",
diff --git a/evals/coding-agent/graders/pipe-store-error.fixture.tsx.txt b/evals/coding-agent/graders/pipe-store-error.fixture.tsx.txt
new file mode 100644
index 000000000..24a13a88c
--- /dev/null
+++ b/evals/coding-agent/graders/pipe-store-error.fixture.tsx.txt
@@ -0,0 +1,132 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import React from "react";
+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { PipeStoreView } from "@/components/pipe-store";
+
+const mocks = vi.hoisted(() => ({
+ localFetch: vi.fn(),
+ cacheSet: vi.fn(),
+ invalidatePrefix: vi.fn(),
+}));
+
+vi.mock("@/lib/api", () => ({ localFetch: mocks.localFetch }));
+vi.mock("@/lib/cache", () => ({
+ apiCache: {
+ get: vi.fn(() => null),
+ getStale: vi.fn(() => null),
+ isFresh: vi.fn(() => false),
+ set: mocks.cacheSet,
+ invalidate: vi.fn(),
+ invalidatePrefix: mocks.invalidatePrefix,
+ },
+}));
+vi.mock("@/lib/hooks/use-settings", () => ({
+ useSettings: () => ({ settings: { user: null } }),
+}));
+vi.mock("@/lib/hooks/use-event-listener", () => ({ useEventListener: vi.fn() }));
+vi.mock("@/components/ui/use-toast", () => ({
+ useToast: () => ({ toast: vi.fn() }),
+}));
+vi.mock("@/components/settings/pipes-section", () => ({
+ PipesSection: () => <div>installed tasks</div>,
+}));
+vi.mock("@/components/notification-bell", () => ({ NotificationBell: () => null }));
+vi.mock("@/components/pipe-store-submission", () => ({
+ PipeStoreSubmissionDialog: () => null,
+}));
+vi.mock("@/lib/stores/feedback-store", () => ({
+ useFeedbackStore: () => vi.fn(),
+}));
+vi.mock("@tauri-apps/plugin-opener", () => ({ openUrl: vi.fn() }));
+vi.mock("posthog-js", () => ({ default: { capture: vi.fn() } }));
+
+function jsonResponse(data: unknown, status = 200): Response {
+ return {
+ ok: status >= 200 && status < 300,
+ status,
+ json: async () => data,
+ } as Response;
+}
+
+describe("pipe store error state", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.spyOn(console, "error").mockImplementation(() => {});
+ const values = new Map<string, string>();
+ vi.stubGlobal("localStorage", {
+ getItem: (key: string) => values.get(key) ?? null,
+ setItem: (key: string, value: string) => values.set(key, value),
+ removeItem: (key: string) => values.delete(key),
+ clear: () => values.clear(),
+ });
+ localStorage.setItem("screenpipe:pipes-welcome-dismissed", "1");
+ });
+
+ afterEach(() => {
+ cleanup();
+ localStorage.clear();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+ });
+
+ const errorText = /(?:couldn.t|cannot|unable to) load|load.*failed|catalog.*unavailable/i;
+ const emptyText = /no (?:scheduled tasks|tasks|pipes|workflows) (?:found|available)|catalog is empty/i;
+ const retryName = /try again|retry|reload/i;
+ const pipe = { slug: "daily-summary", title: "Daily summary", description: "summarizes the day", category: "productivity" };
+ function installResponses(store: () => Promise<Response>) {
+ mocks.localFetch.mockImplementation(async (url: string) => {
+ if (url === "/pipes") return jsonResponse([{ config: { name: "installed" } }]);
+ if (url === "/pipes/store/check-updates") return jsonResponse({ data: [] });
+ if (url.startsWith("/pipes/store?")) return store();
+ throw new Error(`unexpected request: ${url}`);
+ });
+ }
+ function discover() {
+ render(<PipeStoreView />);
+ expect(screen.getByText("installed tasks")).toBeTruthy();
+ fireEvent.click(screen.getByRole("button", { name: /discover/i }));
+ }
+ function noError() {
+ expect(screen.queryAllByText(errorText)).toHaveLength(0);
+ expect(screen.queryByRole("button", { name: retryName })).toBeNull();
+ }
+ for (const failure of ["payload", "http", "network"]) {
+ for (const recovered of ["populated", "empty"]) {
+ it(`${failure} failure stays distinct from empty and retries to ${recovered}`, async () => {
+ let failed = false;
+ installResponses(async () => {
+ if (!failed) {
+ failed = true;
+ if (failure === "network") throw new Error("offline");
+ return failure === "http" ? jsonResponse({}, 503) : jsonResponse({ error: "registry unavailable" });
+ }
+ return jsonResponse({ data: recovered === "empty" ? [] : [pipe] });
+ });
+ discover();
+ await waitFor(() => expect(screen.queryAllByText(errorText).length).toBeGreaterThan(0));
+ expect(screen.queryByText(emptyText)).toBeNull();
+ fireEvent.click(screen.getByRole("button", { name: retryName }));
+ await waitFor(() => expect(screen.getByText(recovered === "empty" ? emptyText : "Daily summary")).toBeTruthy());
+ noError();
+ });
+ }
+ }
+ for (const payload of [[], { data: [] }, { pipes: [] }]) {
+ it(`successful empty catalog stays empty: ${JSON.stringify(payload)}`, async () => {
+ installResponses(async () => jsonResponse(payload));
+ discover();
+ await waitFor(() => expect(screen.getByText(emptyText)).toBeTruthy());
+ noError();
+ });
+ }
+ it("successful populated catalog preserves its content", async () => {
+ installResponses(async () => jsonResponse({ data: [pipe] }));
+ discover();
+ await waitFor(() => expect(screen.getByText("Daily summary")).toBeTruthy());
+ noError();
+ expect(screen.queryByText(emptyText)).toBeNull();
+ });
+});