Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: grade privacy category ownership through settings callers

Louis Beaumontfb5d1b485eab
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 26ac85bb9..ea5a42228 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -123,3 +123,28 @@ complete source review or verified regressions. See [DESIGN.md](./DESIGN.md).
`bun test evals/coding-agent/mine-history.test.ts` runs four synthetic controls
covering old and merged fixes, per-parent merge paths, complete bounded paging,
root/non-fix exclusions, explicit date filtering and invalid inputs.
+
+## Privacy category ownership caller coverage
+
+`app-privacy-category-rule-ownership` executes the historical PrivacySection,
+ContentFiltersCard and category-switch modules, then inspects writes at the
+settings IO boundary across repeated interactions with fresh component state.
+Fifteen outcomes cover manual domain/app exclusions, owned-filter cleanup,
+other-category and include-rule preservation, repeated disable and legacy state.
+The original helper-only fixture accepted an incomplete patch whose UI still
+deleted the user's domain; the replacement rejects that bypass.
+
+Run `bun test evals/coding-agent/calibrate-privacy-ownership.test.js` for eleven
+controls: parent/reference, helper-only patch, missing persistence/reload/card
+forwarding, equivalent callback/component and ownership-field renaming, no-op,
+removed categories and missing-source setup failure. These are deterministic
+grader controls, not agent trials.
+
+The complete source modules are compiled with Bun and executed with synthetic
+React hooks/host elements, identity sanitization and settings IO. Unrelated
+native, telemetry, discovery and visual ports are substituted; native/provider
+actions fail closed. This is not React reconciliation, DOM/browser interaction,
+real settings-store serialization, validation, native capture or current-head
+integration. The adapter retains the existing category-switch UI contract;
+module relocation or a different UI/IO abstraction can require recalibration.
+No dependency link or model call is needed for historical verification.
diff --git a/evals/coding-agent/calibrate-privacy-ownership.test.js b/evals/coding-agent/calibrate-privacy-ownership.test.js
new file mode 100644
index 000000000..ddfb7beec
--- /dev/null
+++ b/evals/coding-agent/calibrate-privacy-ownership.test.js
@@ -0,0 +1,73 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { afterAll, expect, test } from "bun:test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+const repo = resolve(import.meta.dir, "../..");
+const manifest = JSON.parse(readFileSync(join(import.meta.dir, "cases.json"), "utf8"));
+const item = manifest.cases.find(item => item.id === "app-privacy-category-rule-ownership");
+const prefix = "apps/screenpipe-app-tauri/";
+const section = "components/settings/privacy-section.tsx";
+const card = "components/settings/capture-filters/content-filters-card.tsx";
+const switches = "components/settings/capture-filters/category-switches.tsx";
+const helper = "lib/settings/capture-categories.ts";
+const paths = [section, card, switches, helper, "lib/settings/capture-filters.ts"];
+const show = (ref, path) => execFileSync("git", ["-c", "core.commitGraph=false", "show", `${ref}:${prefix}${path}`], { cwd: repo, encoding: "utf8" });
+const broken = Object.fromEntries(paths.map(path => [path, show(item.base_ref, path)]));
+const fixed = Object.fromEntries(paths.map(path => [path, show(item.oracle_ref, path)]));
+const grader = readFileSync(join(import.meta.dir, "graders/app-privacy-ownership.test.js"), "utf8");
+const root = mkdtempSync(join(tmpdir(), "privacy-ownership-calibration-"));
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+function replace(source, old, replacement) {
+ expect(source.split(old)).toHaveLength(2);
+ return source.replace(old, replacement);
+}
+function grade(name, sources) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ for (const [path, source] of Object.entries(sources)) {
+ const file = join(cwd, prefix, path); mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, source);
+ }
+ writeFileSync(join(cwd, "grader.test.js"), grader);
+ return spawnSync(process.execPath, ["test", "grader.test.js"], { cwd, encoding: "utf8", timeout: 30_000, env: { PATH: dirname(process.execPath) } });
+}
+function fails(result) {
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(result.status).toBe(1);
+ expect(result.stderr).toMatch(/expect\(received\)|error: category .+ renders/); expect(result.stderr).not.toContain("Unhandled error");
+ expect(result.stderr).not.toContain("Cannot find module");
+}
+test("parent reaches ownership failures while preserving cleanup and legacy behavior", () => {
+ const result = grade("parent", broken); fails(result);
+ expect(result.stderr).toContain("7 pass"); expect(result.stderr).toContain("8 fail");
+});
+test("historical fix passes every caller outcome", () => {
+ const result = grade("reference", fixed); expect(result.status).toBe(0); expect(result.stderr).toContain("15 pass");
+});
+test("fixed helper alone cannot conceal broken UI wiring", () => fails(grade("helper-only", { ...broken, [helper]: fixed[helper] })));
+test("component wiring without persisting ownership is rejected", () => fails(grade("no-persistence", { ...fixed,
+ [section]: replace(fixed[section], 'categoryOwnedFilters: next.owned ?? { apps: [], domains: [] },', ''),
+})));
+test("persisted ownership must be loaded into the card", () => fails(grade("no-reload", { ...fixed,
+ [section]: replace(fixed[section], 'categoryOwned={settings.categoryOwnedFilters}', ''),
+})));
+test("the card must forward ownership into the actual toggle", () => fails(grade("no-card-forward", { ...fixed, [card]: broken[card] })));
+test("equivalent callback and component renaming is accepted", () => {
+ const sources = Object.fromEntries(Object.entries(fixed).map(([p,s]) => [p, s.replaceAll("handleCategoryToggle", "toggleCaptureGroup").replaceAll("CategoryRow", "CaptureGroupRow").replaceAll("setCategoryEnabled", "toggleCategoryExclusions")]));
+ expect(grade("renamed", sources).status).toBe(0);
+});
+test("equivalent ownership field names remain valid", () => {
+ const sources = Object.fromEntries(Object.entries(fixed).map(([p,s]) => [p, s.replaceAll("categoryOwnedFilters", "captureGroupProvenance").replaceAll("categoryOwned", "captureProvenance").replace(/\bowned\b/g, "provenance")]));
+ expect(grade("ownership-renamed", sources).status).toBe(0);
+});
+test("a no-op toggle cannot pass by preserving all user entries", () => fails(grade("no-op", { ...fixed,
+ [helper]: replace(fixed[helper], 'return enabled ? enableCategory(targets, category) : disableCategory(targets, category);', 'return targets;'),
+})));
+test("removing all categories cannot hide failing scenarios", () => fails(grade("empty-categories", { ...fixed,
+ [switches]: fixed[switches].replaceAll('CAPTURE_CATEGORIES.map', '[].map'),
+})));
+test("missing real source is a setup error, not a behavioral contrast", () => {
+ const sources = { ...fixed }; delete sources[card]; const result = grade("missing-source", sources);
+ expect(result.status).toBe(1); expect(result.stderr).toContain("Cannot find module"); expect(result.stderr).toContain("0 pass");
+});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 2b4b3ea81..b66972218 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-18.2",
+ "dataset_version": "2026-09-19.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -482,24 +482,44 @@
{
"id": "app-privacy-category-rule-ownership",
"name": "Never delete user-owned privacy rules with a category",
- "source": { "kind": "git_regression", "fix_commit": "4450aaa82d" },
- "base_ref": "4450aaa82d^",
- "oracle_ref": "4450aaa82d",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "4450aaa82d30d262a5375520a734f1dff143e81d"
+ },
+ "base_ref": "9ebcd29c6ada189922c02e4ab65f1a08e228e1a1",
+ "oracle_ref": "4450aaa82d30d262a5375520a734f1dff143e81d",
"gate": "advisory",
- "trigger_paths": ["apps/screenpipe-app-tauri/lib/settings/capture-categories.ts"],
- "tags": ["regression", "escaped-failure", "privacy", "settings", "data-loss"],
- "prompt": "Disabling a privacy capture category deletes an identical exclusion rule or domain the user created manually. Track category ownership so disabling removes only entries that category actually created, preserves hand-written and other-category entries, and still cleans up all owned filters. Keep a safe compatibility path for legacy state with no provenance.",
- "agent_timeout_seconds": 900,
- "dependency_links": [
- { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/settings/privacy-section.tsx",
+ "apps/screenpipe-app-tauri/components/settings/capture-filters/content-filters-card.tsx",
+ "apps/screenpipe-app-tauri/lib/settings/capture-categories.ts",
+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "privacy",
+ "settings",
+ "data-loss"
],
+ "prompt": "Disabling a privacy capture category deletes an identical exclusion rule or domain the user created manually. Fix the actual Settings > Privacy category switches so enabling then disabling preserves hand-written exclusions and other categories, including after closing and reopening settings. Persist whatever ownership information the UI needs. Still add the category filters when enabled, clean up all filters it created when disabled, preserve unrelated include rules, and tolerate repeated disable. Keep a safe compatibility path so legacy enabled categories without provenance can still be disabled. No particular helper, callback or ownership field name is required. Preserve the existing category choices and privacy controls.",
+ "agent_timeout_seconds": 900,
"grader": {
"timeout_seconds": 300,
"fixtures": [
- { "source_path": "apps/screenpipe-app-tauri/lib/settings/__tests__/capture-categories.test.ts" }
+ {
+ "local_path": "graders/app-privacy-ownership.test.js",
+ "destination_path": "evals/coding-agent/graders/app-privacy-ownership.test.js"
+ }
],
- "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts lib/settings/__tests__/capture-categories.test.ts"
- }
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/app-privacy-ownership.test.js"
+ },
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/settings/privacy-section.tsx",
+ "apps/screenpipe-app-tauri/components/settings/capture-filters/content-filters-card.tsx",
+ "apps/screenpipe-app-tauri/lib/settings/capture-categories.ts",
+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx"
+ ]
},
{
"id": "app-acp-explicit-full-access-boundary",
diff --git a/evals/coding-agent/graders/app-privacy-ownership.test.js b/evals/coding-agent/graders/app-privacy-ownership.test.js
new file mode 100644
index 000000000..dcf56a941
--- /dev/null
+++ b/evals/coding-agent/graders/app-privacy-ownership.test.js
@@ -0,0 +1,231 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { expect, test } from "bun:test";
+import { existsSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { runInNewContext } from "node:vm";
+
+// Execute complete historical component modules. React hooks/host elements,
+// settings IO and unrelated UI/native ports are synthetic; no desktop is run.
+const app = resolve("apps/screenpipe-app-tauri");
+const realPaths = [
+ "components/settings/privacy-section.tsx",
+ "components/settings/capture-filters/content-filters-card.tsx",
+ "components/settings/capture-filters/category-switches.tsx",
+ "lib/settings/capture-categories.ts",
+ "lib/settings/capture-filters.ts",
+];
+const built = new Map();
+for (const path of realPaths) {
+ const filename = resolve(app, path);
+ if (!existsSync(filename)) throw new Error(`Cannot find module '${filename}'`);
+ const result = await Bun.build({ entrypoints: [filename], external: ["*"], target: "node", format: "cjs", write: false });
+ if (!result.success) throw new Error(`Cannot compile privacy source: ${result.logs.join("\n")}`);
+ built.set(filename, await result.outputs[0].text());
+}
+
+function environment(initial = {}) {
+ let settings = { ignoredWindows: [], includedWindows: [], ignoredUrls: [], ...structuredClone(initial) };
+ const writes = [];
+ const cache = new Map();
+ const element = (type, props, ...children) => ({ type, props: { ...props, ...(children.length ? { children } : {}) } });
+ const react = {
+ createElement: element, Fragment: "fragment",
+ useState: initial => [typeof initial === "function" ? initial() : initial, () => {}],
+ useRef: value => ({ current: value }), useMemo: fn => fn(), useCallback: fn => fn,
+ // Effects load native/telemetry state unrelated to category changes.
+ useEffect: () => {},
+ };
+ const host = new Proxy({}, { get: (_, name) => name === "__esModule" ? true : String(name) });
+ const forbidden = () => { throw new Error("Unexpected native/provider side effect"); };
+ function port(id, from) {
+ const path = id.startsWith("@/") ? resolve(app, id.slice(2)) : id.startsWith(".") ? resolve(dirname(from), id) : id;
+ for (const suffix of ["", ".ts", ".tsx"]) if (built.has(path + suffix)) return load(path + suffix);
+ if (id === "react") return { ...react, default: react };
+ if (id === "react/jsx-runtime" || id === "react/jsx-dev-runtime") return { jsx: (type, props) => element(type, props), jsxs: (type, props) => element(type, props), jsxDEV: (type, props) => element(type, props), Fragment: "fragment" };
+ if (id === "@/lib/hooks/use-settings") return { useSettings: () => ({ settings, updateSettings: async patch => {
+ writes.push(structuredClone(patch)); settings = { ...settings, ...structuredClone(patch) };
+ } }) };
+ if (id === "@/lib/hooks/use-managed-policy") return { useManagedPolicy: () => ({ getManagedValue: () => undefined, isManagedDeployment: false }) };
+ if (id === "@/components/ui/use-toast") return { useToast: () => ({ toast: forbidden }) };
+ if (id === "@/lib/hooks/use-sql-autocomplete") return { useSqlAutocomplete: () => ({ items: [], isLoading: false }) };
+ if (id === "@/lib/hooks/use-installed-apps") return { useInstalledApps: () => ({ apps: [] }) };
+ if (id === "@/lib/utils/validation") return { sanitizeValue: (_, value) => value, validateField: () => ({ isValid: true }), debounce: fn => fn };
+ if (id === "@/lib/utils") return { cn: (...parts) => parts.filter(Boolean).join(" ") };
+ if (id === "@/lib/web-url") return { screenpipeWebUrl: () => "https://example.invalid" };
+ if (id === "./icon-urls") return { appIconUrl: () => "" };
+ if (id === "@tauri-apps/plugin-os") return { platform: () => "linux" };
+ if (id === "@/lib/utils/tauri") return { commands: new Proxy({}, { get: () => forbidden }) };
+ if (id === "posthog-js") return { default: new Proxy({}, { get: () => forbidden }) };
+ if (["@sentry/react", "tauri-plugin-sentry-api", "@tauri-apps/plugin-shell", "@/lib/utils/incognito-permission", "@/lib/analytics-id"].includes(id)) return new Proxy({}, { get: () => forbidden });
+ // Only visual leaves outside the ownership path are opaque host elements.
+ if (id === "lucide-react" || id.startsWith("@/components/") || (id.startsWith(".") && from.includes("/components/"))) return host;
+ throw new Error(`Cannot find module '${id}' in synthetic privacy adapter`);
+ }
+ function load(filename) {
+ if (cache.has(filename)) return cache.get(filename).exports;
+ const module = { exports: {} }; cache.set(filename, module);
+ runInNewContext(built.get(filename), { module, exports: module.exports, require: id => port(id, filename), console, process: { env: {} } }, { filename });
+ return module.exports;
+ }
+ const { PrivacySection } = load(resolve(app, realPaths[0]));
+ const categories = load(resolve(app, realPaths[3])).CAPTURE_CATEGORIES;
+ function expand(node) {
+ if (!node || typeof node !== "object") return node;
+ if (Array.isArray(node)) return node.map(expand);
+ if (typeof node.type === "function") {
+ return expand(node.type(node.props));
+ }
+ return { ...node, props: { ...node.props, children: expand(node.props?.children) } };
+ }
+ function find(node, predicate) {
+ if (!node || typeof node !== "object") return undefined;
+ if (Array.isArray(node)) return node.map(n => find(n, predicate)).find(Boolean);
+ return predicate(node) ? node : find(node.props?.children, predicate);
+ }
+ return {
+ categories, writes, settings: () => structuredClone(settings),
+ async toggle(id, enabled) {
+ const tree = expand(PrivacySection());
+ const row = find(tree, n => n.props?.["data-category"] === id);
+ expect(row, `category ${id} renders`).toBeDefined();
+ const control = find(row, n => typeof n.props?.onCheckedChange === "function");
+ expect(control, "category switch has an event handler").toBeDefined();
+ const count = writes.length; await control.props.onCheckedChange(enabled);
+ // The real settings callback serializes writes through a Promise chain.
+ await new Promise(resolve => setTimeout(resolve, 0));
+ expect(writes.length).toBeGreaterThan(count);
+ // Discard all component state between interactions, like reopening settings.
+ },
+ };
+}
+
+// Freeze task inputs independently of candidate category definitions.
+const definitions = [
+ {
+ "id": "password-managers",
+ "apps": [
+ "1Password::",
+ "Bitwarden::",
+ "LastPass::",
+ "Dashlane::",
+ "KeePassXC::",
+ "KeePass::",
+ "NordPass::",
+ "Enpass::",
+ "Proton Pass::",
+ "Keychain Access::",
+ "Credential Manager::"
+ ],
+ "domains": [
+ "1password.com",
+ "bitwarden.com",
+ "lastpass.com",
+ "dashlane.com",
+ "keepersecurity.com",
+ "nordpass.com",
+ "pass.proton.me"
+ ]
+ },
+ {
+ "id": "personal-messaging",
+ "apps": [
+ "WhatsApp::",
+ "Signal::",
+ "Telegram::",
+ "Messages::",
+ "Messenger::"
+ ],
+ "domains": [
+ "web.whatsapp.com",
+ "web.telegram.org",
+ "messenger.com"
+ ]
+ },
+ {
+ "id": "banking-finance",
+ "apps": [],
+ "domains": [
+ "chase.com",
+ "bankofamerica.com",
+ "wellsfargo.com",
+ "citi.com",
+ "capitalone.com",
+ "schwab.com",
+ "fidelity.com",
+ "vanguard.com",
+ "paypal.com",
+ "wise.com",
+ "revolut.com",
+ "coinbase.com"
+ ]
+ },
+ {
+ "id": "health",
+ "apps": [],
+ "domains": [
+ "mychart.com",
+ "healthcare.gov",
+ "cvs.com",
+ "walgreens.com",
+ "zocdoc.com"
+ ]
+ },
+ {
+ "id": "media",
+ "apps": [
+ "Netflix::",
+ "Spotify::",
+ "Steam::"
+ ],
+ "domains": [
+ "netflix.com",
+ "hulu.com",
+ "disneyplus.com",
+ "max.com",
+ "twitch.tv",
+ "open.spotify.com"
+ ]
+ }
+];
+for (const category of definitions) {
+ if (category.domains.length) test(`${category.id}: preserve manually supplied domains across enable, persisted reload and disable`, async () => {
+ const manual = category.domains[0]; if (!manual) return;
+ const env = environment({ ignoredUrls: [manual, "manual.example.invalid"] });
+ await env.toggle(category.id, true);
+ for (const domain of category.domains) expect(env.settings().ignoredUrls).toContain(domain);
+ await env.toggle(category.id, false);
+ expect(env.settings().ignoredUrls.sort()).toEqual([manual, "manual.example.invalid"].sort());
+ expect(env.settings().ignoredWindows).toEqual([]);
+ });
+ if (category.apps.length) test(`${category.id}: preserve manually supplied scoped app exclusion`, async () => {
+ const manual = category.apps[0]; if (!manual) return;
+ const env = environment({ ignoredWindows: [manual, "Handwritten::private"] });
+ await env.toggle(category.id, true); await env.toggle(category.id, false);
+ expect(env.settings().ignoredWindows.sort()).toEqual([manual, "Handwritten::private"].sort());
+ expect(env.settings().ignoredUrls).toEqual([]);
+ });
+ test(`${category.id}: clean up created filters and tolerate repeated off`, async () => {
+ const env = environment(); await env.toggle(category.id, true);
+ for (const app of category.apps) expect(env.settings().ignoredWindows).toContain(app);
+ for (const domain of category.domains) expect(env.settings().ignoredUrls).toContain(domain);
+ await env.toggle(category.id, false); await env.toggle(category.id, false);
+ expect(env.settings().ignoredWindows).toEqual([]); expect(env.settings().ignoredUrls).toEqual([]);
+ });
+}
+test("turning one category off preserves another category and unrelated include rules", async () => {
+ const [first, second] = definitions; const env = environment({ includedWindows: ["Work::meeting"] });
+ await env.toggle(first.id, true); await env.toggle(second.id, true); await env.toggle(first.id, false);
+ for (const app of second.apps) expect(env.settings().ignoredWindows).toContain(app);
+ for (const domain of second.domains) expect(env.settings().ignoredUrls).toContain(domain);
+ expect(env.settings().includedWindows).toEqual(["Work::meeting"]);
+ await env.toggle(second.id, false);
+ expect(env.settings().ignoredWindows).toEqual([]); expect(env.settings().ignoredUrls).toEqual([]);
+});
+test("legacy enabled category without provenance can still be disabled", async () => {
+ const first = definitions[0]; const env = environment({ ignoredWindows: [...first.apps, "Manual::private"], ignoredUrls: [...first.domains, "manual.example.invalid"] });
+ await env.toggle(first.id, false);
+ expect(env.settings().ignoredWindows).toEqual(["Manual::private"]);
+ expect(env.settings().ignoredUrls).toEqual(["manual.example.invalid"]);
+});