Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: catch partial shared-log privacy fixes (#7736)

Louis Beaumont5c2744fa2537
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index d72c475fe..3b851fac7 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,20 @@
# Coding-agent regression evals
+`app-log-share-redaction-false-positive` mounts the historical shared-log upload
+caller with synthetic chat and native/network ports. Eight outcomes preserve
+numeric telemetry and chat/log context while checking formatted phones, personal
+data, payment cards, API credentials, non-local IP addresses and home usernames.
+The parent fails flattened bounds and complete card removal, preserving six; the reference
+passes eight. Replacement labels and inline redactors remain valid solutions.
+Run `bun test evals/coding-agent/calibrate-log-share-redaction.test.js` with desktop
+Vitest dependencies available. Twelve controls reject unused fixes, disabled or
+partial redaction, omitted chat and corrupted telemetry; missing source remains
+an infrastructure error. `SCREENPIPE_EVAL_DESKTOP_DEPENDENCIES` can select an
+existing dependency directory for calibration. Fixtures and dependencies appear
+only at grading. The current upload path uses native feedback jobs, so these
+historical outcomes do not establish current native redaction, live upload,
+complete privacy coverage, enforced isolation or model performance.
+
`app-audio-drop-control-dispatch` executes the actual AudioStream destructor
with synthetic streams and no capture hardware. Three Drop outcomes distinguish
the broken parent from the reference while two checks preserve live-stream and
diff --git a/evals/coding-agent/calibrate-log-share-redaction.test.js b/evals/coding-agent/calibrate-log-share-redaction.test.js
new file mode 100644
index 000000000..1d9e2dab9
--- /dev/null
+++ b/evals/coding-agent/calibrate-log-share-redaction.test.js
@@ -0,0 +1,54 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from "bun:test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, writeFileSync, symlinkSync, rmSync, existsSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { classifyGraderError } from "./grader-outcome.mjs";
+const repo = resolve(import.meta.dir, "../.."), app = "apps/screenpipe-app-tauri";
+const item = JSON.parse(readFileSync(join(import.meta.dir, "cases.json"), "utf8")).cases.find(c => c.id === "app-log-share-redaction-false-positive");
+const component = `${app}/components/share-logs-button.tsx`, helper = `${app}/lib/utils/redact-pii.ts`;
+const deps = resolve(process.env.SCREENPIPE_EVAL_DESKTOP_DEPENDENCIES || join(repo, app, "node_modules"));
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+const root = mkdtempSync(join(tmpdir(), "log-share-calibration-"));
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+function git(...args) { return execFileSync("git", args, { cwd: repo, maxBuffer: 64 * 1024 * 1024 }); }
+function edit(cwd, path, transform) {
+ const file = join(cwd, path), before = readFileSync(file, "utf8"), after = transform(before);
+ expect(after).not.toBe(before); writeFileSync(file, after);
+}
+function removeSection(source, start, end) {
+ const a = source.indexOf(start), b = source.indexOf(end, a);
+ expect(a).toBeGreaterThan(-1); expect(b).toBeGreaterThan(a);
+ return source.slice(0, a) + source.slice(b);
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ const paths = ["components", "lib", "types", "vitest.config.ts", "vitest.setup.ts", "tsconfig.json", "package.json"].map(p => `${app}/${p}`).filter(p => git("ls-tree", ref, "--", p).toString().trim());
+ execFileSync("tar", ["-x", "-C", cwd], { input: git("archive", ref, "--", ...paths) });
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), { recursive: true }); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ expect(existsSync(join(deps, "vitest/package.json"))).toBe(true);
+ symlinkSync(deps, join(cwd, app, "node_modules"), "dir"); mutate(cwd);
+ const result = spawnSync("/bin/bash", ["-c", item.grader.command], { cwd, encoding: "utf8", timeout: 60000, maxBuffer: 4 * 1024 * 1024, env: { PATH: process.env.PATH, HOME: cwd, CI: "true", TZ: "UTC", NO_COLOR: "1" } });
+ const kind = classifyGraderError(result);
+ if (receipts) { mkdirSync(receipts, { recursive: true }); for (const channel of ["stdout", "stderr"]) writeFileSync(join(receipts, `${name}.${channel}`), result[channel] || ""); writeFileSync(join(receipts, `${name}.json`), JSON.stringify({ ref, command: item.grader.command, exit_code: result.status, signal: result.signal, error: result.error?.message, grader_error_kind: kind }, null, 2)); }
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); return { ...result, kind };
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.kind).toBeNull(); expect(r.stdout).toContain("8 passed"); }
+function fails(r) { expect(r.status).toBe(1); expect(r.kind).toBeNull(); expect(r.stderr).toMatch(/AssertionError|Error: expect\(/); }
+test("historical parent fails intended outcomes", () => { const r = grade("parent", item.base_ref); fails(r); expect(r.stdout).toContain("2 failed | 6 passed"); expect(r.stdout).toContain("bounds_flat=47692 111658 44767 39496"); }, 60000);
+test("historical reference preserves all eight outcomes", () => passes(grade("reference")), 60000);
+test("equivalent replacement labels pass", () => passes(grade("labels", item.oracle_ref, cwd => edit(cwd, helper, s => s.replace(/\[(EMAIL|PHONE|SSN|CARD|REDACTED|IP|USER)\]/g, "[FILTERED]")))), 60000);
+test("equivalent inline redactor passes", () => passes(grade("inline", item.oracle_ref, cwd => {
+ const implementation = git("show", `${item.oracle_ref}:${helper}`).toString().replace("export function", "function");
+ edit(cwd, component, s => s.replace('import { redactPii } from "@/lib/utils/redact-pii";', implementation));
+})), 60000);
+test("unused correct source cannot mask the broken caller", () => fails(grade("unused", item.base_ref, cwd => { writeFileSync(join(cwd, app, "unused-redactor.ts"), git("show", `${item.oracle_ref}:${helper}`)); })), 60000);
+test("API credential leaks are rejected", () => fails(grade("api", item.oracle_ref, cwd => edit(cwd, helper, s => removeSection(s, " // API keys", " // bearer tokens")))), 60000);
+test("non-local IP leaks are rejected", () => fails(grade("ip", item.oracle_ref, cwd => edit(cwd, helper, s => removeSection(s, " // IP addresses", " // home directory")))), 60000);
+test("home-directory username leaks are rejected", () => fails(grade("home", item.oracle_ref, cwd => edit(cwd, helper, s => s.slice(0, s.indexOf(" // home directory")) + ";\n}\n"))), 60000);
+test("corrupting preserved numeric telemetry is rejected", () => fails(grade("numbers", item.oracle_ref, cwd => edit(cwd, helper, s => s.replace("return text", 'return text.replace(/605818409/g, "[FILTERED]")')))), 60000);
+test("omitting chat context is rejected", () => fails(grade("omission", item.oracle_ref, cwd => edit(cwd, component, s => s.replace("const conversations = await loadAllConversations();", "const conversations = [];")))), 60000);
+test("disabling all redaction is rejected", () => fails(grade("disabled", item.oracle_ref, cwd => edit(cwd, helper, s => s.replace("return text", "return text; return text")))), 60000);
+test("missing caller remains an infrastructure error", () => { const r = grade("missing", item.oracle_ref, cwd => rmSync(join(cwd, component))); expect(r.status).toBe(1); expect(r.kind).toBe("vitest_collection_error"); expect(r.stderr).not.toMatch(/^AssertionError/m); }, 60000);
diff --git a/evals/coding-agent/graders/app-log-share-redaction.test.tsx b/evals/coding-agent/graders/app-log-share-redaction.test.tsx
index 19c46bc10..2cc1ac267 100644
--- a/evals/coding-agent/graders/app-log-share-redaction.test.tsx
+++ b/evals/coding-agent/graders/app-log-share-redaction.test.tsx
@@ -65,7 +65,7 @@ describe("shared-log PII redaction outcome", () => {
{
role: "user",
content:
- 'bounds={"left": 47692, "top": 111658, "width": 44767, "height": 39496}; bounds_flat=47692 111658 44767 39496; frame_id=549130407; hash=5542957767813040527; display=1920x1080; [email protected]; phone=(555) 123-4567; ssn=123-45-6789; card=4532-1234-5678-9012; Bearer abcdefghijklmnopqrstuvwxyz; path=/Users/example/.screenpipe',
+ 'bounds={"left": 47692, "top": 111658, "width": 44767, "height": 39496}; bounds_flat=47692 111658 44767 39496; frame_id=549130407; hash=5542957767813040527; display=1920x1080; monitor_id=605818409; pid=25616; elapsed=100.4287155s; [email protected]; phone=(555) 123-4567; ssn=123-45-6789; card=4532-1234-5678-9012; Bearer abcdefghijklmnopqrstuvwxyz; path=/Users/example/.screenpipe; api_key=synthetic_credential_abcdefgh ; password=synthetic_password_abcdefgh ; remote=203.0.113.29; linux=/home/synthetic-person/.config; phones=234-567-8901,+1-234-567-8901' + String.raw`; windows=C:\Users\synthetic-person\.screenpipe`,
},
],
},
@@ -103,10 +103,18 @@ describe("shared-log PII redaction outcome", () => {
vi.clearAllMocks();
});
+ function field(name: string) {
+ const match = uploadedBody.match(new RegExp(`(?:^|[;\\n])\\s*${name}=([^;\\n]*)`));
+ expect(match, `missing ${name} context`).not.toBeNull();
+ return match![1];
+ }
+
async function uploadSyntheticBundle() {
render(<ShareLogsButton />);
fireEvent.click(screen.getByRole("button", { name: /send logs & feedback/i }));
await waitFor(() => expect(uploadedBody).toContain("Pi Chat History"));
+ expect(uploadedBody).toContain("Synthetic troubleshooting");
+ expect(uploadedBody).toContain("ordinary log line");
}
it("preserves structured bounds and pixel dimensions", async () => {
@@ -124,24 +132,62 @@ describe("shared-log PII redaction outcome", () => {
expect(uploadedBody).toContain("bounds_flat=47692 111658 44767 39496");
expect(uploadedBody).toContain("frame_id=549130407");
expect(uploadedBody).toContain("hash=5542957767813040527");
+ expect(uploadedBody).toContain("monitor_id=605818409");
+ expect(uploadedBody).toContain("pid=25616");
+ expect(uploadedBody).toContain("elapsed=100.4287155s");
});
it("continues to redact common personal data", async () => {
await uploadSyntheticBundle();
- expect(uploadedBody).toContain("contact=[EMAIL]");
- expect(uploadedBody).toContain("phone=[PHONE]");
- expect(uploadedBody).toContain("ssn=[SSN]");
- expect(uploadedBody).toContain("Bearer [REDACTED]");
- expect(uploadedBody).toContain("path=/Users/[USER]/.screenpipe");
expect(uploadedBody).not.toContain("[email protected]");
expect(uploadedBody).not.toContain("(555) 123-4567");
+ expect(uploadedBody).not.toContain("123-45-6789");
+ expect(uploadedBody).not.toContain("abcdefghijklmnopqrstuvwxyz");
+ expect(uploadedBody).not.toContain("/Users/example/");
+ expect(uploadedBody).toContain("contact=");
+ expect(uploadedBody).toContain("phone=");
+ expect(field("phone")).not.toMatch(/\d/);
+ expect(field("ssn")).not.toMatch(/\d/);
});
it("does not let phone matching corrupt payment-card redaction", async () => {
await uploadSyntheticBundle();
- expect(uploadedBody).toContain("card=[CARD]");
expect(uploadedBody).not.toContain("4532-1234-5678-9012");
+ expect(field("card")).not.toMatch(/\d/);
+ });
+
+ it("removes API credentials without requiring a particular replacement label", async () => {
+ await uploadSyntheticBundle();
+ expect(uploadedBody).toContain("api_key=");
+ expect(uploadedBody).toContain("password=");
+ expect(uploadedBody).not.toContain("synthetic_credential_abcdefgh");
+ expect(uploadedBody).not.toContain("synthetic_password_abcdefgh");
+ for (const name of ["api_key", "password"]) {
+ expect(field(name)).not.toMatch(/synthetic|credential|password|abcdefgh/);
+ }
+ });
+
+ it("removes non-local IP addresses while retaining their surrounding context", async () => {
+ await uploadSyntheticBundle();
+ expect(uploadedBody).toContain("remote=");
+ expect(uploadedBody).not.toContain("203.0.113.29");
+ expect(field("remote")).not.toMatch(/\d/);
+ });
+
+ it("removes home-directory usernames across supported path formats", async () => {
+ await uploadSyntheticBundle();
+ expect(uploadedBody).toContain("linux=");
+ expect(uploadedBody).toContain("windows=");
+ expect(uploadedBody).not.toContain("synthetic-person");
+ });
+
+ it("keeps redacting dashed and international phone numbers", async () => {
+ await uploadSyntheticBundle();
+ expect(uploadedBody).toContain("phones=");
+ expect(uploadedBody).not.toContain("234-567-8901");
+ expect(uploadedBody).not.toContain("+1-234-567-8901");
+ expect(field("phones")).not.toMatch(/\d/);
});
});