YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: catch partial shared-log privacy fixes (#7736)
Louis Beaumont5c2744fa2537
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex d72c475fe..3b851fac7 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1,5 +1,20 @@# Coding-agent regression evals+`app-log-share-redaction-false-positive` mounts the historical shared-log upload+caller with synthetic chat and native/network ports. Eight outcomes preserve+numeric telemetry and chat/log context while checking formatted phones, personal+data, payment cards, API credentials, non-local IP addresses and home usernames.+The parent fails flattened bounds and complete card removal, preserving six; the reference+passes eight. Replacement labels and inline redactors remain valid solutions.+Run `bun test evals/coding-agent/calibrate-log-share-redaction.test.js` with desktop+Vitest dependencies available. Twelve controls reject unused fixes, disabled or+partial redaction, omitted chat and corrupted telemetry; missing source remains+an infrastructure error. `SCREENPIPE_EVAL_DESKTOP_DEPENDENCIES` can select an+existing dependency directory for calibration. Fixtures and dependencies appear+only at grading. The current upload path uses native feedback jobs, so these+historical outcomes do not establish current native redaction, live upload,+complete privacy coverage, enforced isolation or model performance.+`app-audio-drop-control-dispatch` executes the actual AudioStream destructorwith synthetic streams and no capture hardware. Three Drop outcomes distinguishthe broken parent from the reference while two checks preserve live-stream anddiff --git a/evals/coding-agent/calibrate-log-share-redaction.test.js b/evals/coding-agent/calibrate-log-share-redaction.test.jsnew file mode 100644index 000000000..1d9e2dab9--- /dev/null+++ b/evals/coding-agent/calibrate-log-share-redaction.test.js@@ -0,0 +1,54 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from "bun:test";+import { execFileSync, spawnSync } from "node:child_process";+import { mkdirSync, mkdtempSync, readFileSync, writeFileSync, symlinkSync, rmSync, existsSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { tmpdir } from "node:os";+import { classifyGraderError } from "./grader-outcome.mjs";+const repo = resolve(import.meta.dir, "../.."), app = "apps/screenpipe-app-tauri";+const item = JSON.parse(readFileSync(join(import.meta.dir, "cases.json"), "utf8")).cases.find(c => c.id === "app-log-share-redaction-false-positive");+const component = `${app}/components/share-logs-button.tsx`, helper = `${app}/lib/utils/redact-pii.ts`;+const deps = resolve(process.env.SCREENPIPE_EVAL_DESKTOP_DEPENDENCIES || join(repo, app, "node_modules"));+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+const root = mkdtempSync(join(tmpdir(), "log-share-calibration-"));+afterAll(() => rmSync(root, { recursive: true, force: true }));+function git(...args) { return execFileSync("git", args, { cwd: repo, maxBuffer: 64 * 1024 * 1024 }); }+function edit(cwd, path, transform) {+ const file = join(cwd, path), before = readFileSync(file, "utf8"), after = transform(before);+ expect(after).not.toBe(before); writeFileSync(file, after);+}+function removeSection(source, start, end) {+ const a = source.indexOf(start), b = source.indexOf(end, a);+ expect(a).toBeGreaterThan(-1); expect(b).toBeGreaterThan(a);+ return source.slice(0, a) + source.slice(b);+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ const paths = ["components", "lib", "types", "vitest.config.ts", "vitest.setup.ts", "tsconfig.json", "package.json"].map(p => `${app}/${p}`).filter(p => git("ls-tree", ref, "--", p).toString().trim());+ execFileSync("tar", ["-x", "-C", cwd], { input: git("archive", ref, "--", ...paths) });+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), { recursive: true }); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ expect(existsSync(join(deps, "vitest/package.json"))).toBe(true);+ symlinkSync(deps, join(cwd, app, "node_modules"), "dir"); mutate(cwd);+ const result = spawnSync("/bin/bash", ["-c", item.grader.command], { cwd, encoding: "utf8", timeout: 60000, maxBuffer: 4 * 1024 * 1024, env: { PATH: process.env.PATH, HOME: cwd, CI: "true", TZ: "UTC", NO_COLOR: "1" } });+ const kind = classifyGraderError(result);+ if (receipts) { mkdirSync(receipts, { recursive: true }); for (const channel of ["stdout", "stderr"]) writeFileSync(join(receipts, `${name}.${channel}`), result[channel] || ""); writeFileSync(join(receipts, `${name}.json`), JSON.stringify({ ref, command: item.grader.command, exit_code: result.status, signal: result.signal, error: result.error?.message, grader_error_kind: kind }, null, 2)); }+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); return { ...result, kind };+}+function passes(r) { expect(r.status).toBe(0); expect(r.kind).toBeNull(); expect(r.stdout).toContain("8 passed"); }+function fails(r) { expect(r.status).toBe(1); expect(r.kind).toBeNull(); expect(r.stderr).toMatch(/AssertionError|Error: expect\(/); }+test("historical parent fails intended outcomes", () => { const r = grade("parent", item.base_ref); fails(r); expect(r.stdout).toContain("2 failed | 6 passed"); expect(r.stdout).toContain("bounds_flat=47692 111658 44767 39496"); }, 60000);+test("historical reference preserves all eight outcomes", () => passes(grade("reference")), 60000);+test("equivalent replacement labels pass", () => passes(grade("labels", item.oracle_ref, cwd => edit(cwd, helper, s => s.replace(/\[(EMAIL|PHONE|SSN|CARD|REDACTED|IP|USER)\]/g, "[FILTERED]")))), 60000);+test("equivalent inline redactor passes", () => passes(grade("inline", item.oracle_ref, cwd => {+ const implementation = git("show", `${item.oracle_ref}:${helper}`).toString().replace("export function", "function");+ edit(cwd, component, s => s.replace('import { redactPii } from "@/lib/utils/redact-pii";', implementation));+})), 60000);+test("unused correct source cannot mask the broken caller", () => fails(grade("unused", item.base_ref, cwd => { writeFileSync(join(cwd, app, "unused-redactor.ts"), git("show", `${item.oracle_ref}:${helper}`)); })), 60000);+test("API credential leaks are rejected", () => fails(grade("api", item.oracle_ref, cwd => edit(cwd, helper, s => removeSection(s, " // API keys", " // bearer tokens")))), 60000);+test("non-local IP leaks are rejected", () => fails(grade("ip", item.oracle_ref, cwd => edit(cwd, helper, s => removeSection(s, " // IP addresses", " // home directory")))), 60000);+test("home-directory username leaks are rejected", () => fails(grade("home", item.oracle_ref, cwd => edit(cwd, helper, s => s.slice(0, s.indexOf(" // home directory")) + ";\n}\n"))), 60000);+test("corrupting preserved numeric telemetry is rejected", () => fails(grade("numbers", item.oracle_ref, cwd => edit(cwd, helper, s => s.replace("return text", 'return text.replace(/605818409/g, "[FILTERED]")')))), 60000);+test("omitting chat context is rejected", () => fails(grade("omission", item.oracle_ref, cwd => edit(cwd, component, s => s.replace("const conversations = await loadAllConversations();", "const conversations = [];")))), 60000);+test("disabling all redaction is rejected", () => fails(grade("disabled", item.oracle_ref, cwd => edit(cwd, helper, s => s.replace("return text", "return text; return text")))), 60000);+test("missing caller remains an infrastructure error", () => { const r = grade("missing", item.oracle_ref, cwd => rmSync(join(cwd, component))); expect(r.status).toBe(1); expect(r.kind).toBe("vitest_collection_error"); expect(r.stderr).not.toMatch(/^AssertionError/m); }, 60000);diff --git a/evals/coding-agent/graders/app-log-share-redaction.test.tsx b/evals/coding-agent/graders/app-log-share-redaction.test.tsxindex 19c46bc10..2cc1ac267 100644--- a/evals/coding-agent/graders/app-log-share-redaction.test.tsx+++ b/evals/coding-agent/graders/app-log-share-redaction.test.tsx@@ -65,7 +65,7 @@ describe("shared-log PII redaction outcome", () => {{role: "user",content:- 'bounds={"left": 47692, "top": 111658, "width": 44767, "height": 39496}; bounds_flat=47692 111658 44767 39496; frame_id=549130407; hash=5542957767813040527; display=1920x1080; [email protected]; phone=(555) 123-4567; ssn=123-45-6789; card=4532-1234-5678-9012; Bearer abcdefghijklmnopqrstuvwxyz; path=/Users/example/.screenpipe',+ 'bounds={"left": 47692, "top": 111658, "width": 44767, "height": 39496}; bounds_flat=47692 111658 44767 39496; frame_id=549130407; hash=5542957767813040527; display=1920x1080; monitor_id=605818409; pid=25616; elapsed=100.4287155s; [email protected]; phone=(555) 123-4567; ssn=123-45-6789; card=4532-1234-5678-9012; Bearer abcdefghijklmnopqrstuvwxyz; path=/Users/example/.screenpipe; api_key=synthetic_credential_abcdefgh ; password=synthetic_password_abcdefgh ; remote=203.0.113.29; linux=/home/synthetic-person/.config; phones=234-567-8901,+1-234-567-8901' + String.raw`; windows=C:\Users\synthetic-person\.screenpipe`,},],},@@ -103,10 +103,18 @@ describe("shared-log PII redaction outcome", () => {vi.clearAllMocks();});+ function field(name: string) {+ const match = uploadedBody.match(new RegExp(`(?:^|[;\\n])\\s*${name}=([^;\\n]*)`));+ expect(match, `missing ${name} context`).not.toBeNull();+ return match![1];+ }+async function uploadSyntheticBundle() {render(<ShareLogsButton />);fireEvent.click(screen.getByRole("button", { name: /send logs & feedback/i }));await waitFor(() => expect(uploadedBody).toContain("Pi Chat History"));+ expect(uploadedBody).toContain("Synthetic troubleshooting");+ expect(uploadedBody).toContain("ordinary log line");}it("preserves structured bounds and pixel dimensions", async () => {@@ -124,24 +132,62 @@ describe("shared-log PII redaction outcome", () => {expect(uploadedBody).toContain("bounds_flat=47692 111658 44767 39496");expect(uploadedBody).toContain("frame_id=549130407");expect(uploadedBody).toContain("hash=5542957767813040527");+ expect(uploadedBody).toContain("monitor_id=605818409");+ expect(uploadedBody).toContain("pid=25616");+ expect(uploadedBody).toContain("elapsed=100.4287155s");});it("continues to redact common personal data", async () => {await uploadSyntheticBundle();- expect(uploadedBody).toContain("contact=[EMAIL]");- expect(uploadedBody).toContain("phone=[PHONE]");- expect(uploadedBody).toContain("ssn=[SSN]");- expect(uploadedBody).toContain("Bearer [REDACTED]");- expect(uploadedBody).toContain("path=/Users/[USER]/.screenpipe");expect(uploadedBody).not.toContain("[email protected]");expect(uploadedBody).not.toContain("(555) 123-4567");+ expect(uploadedBody).not.toContain("123-45-6789");+ expect(uploadedBody).not.toContain("abcdefghijklmnopqrstuvwxyz");+ expect(uploadedBody).not.toContain("/Users/example/");+ expect(uploadedBody).toContain("contact=");+ expect(uploadedBody).toContain("phone=");+ expect(field("phone")).not.toMatch(/\d/);+ expect(field("ssn")).not.toMatch(/\d/);});it("does not let phone matching corrupt payment-card redaction", async () => {await uploadSyntheticBundle();- expect(uploadedBody).toContain("card=[CARD]");expect(uploadedBody).not.toContain("4532-1234-5678-9012");+ expect(field("card")).not.toMatch(/\d/);+ });++ it("removes API credentials without requiring a particular replacement label", async () => {+ await uploadSyntheticBundle();+ expect(uploadedBody).toContain("api_key=");+ expect(uploadedBody).toContain("password=");+ expect(uploadedBody).not.toContain("synthetic_credential_abcdefgh");+ expect(uploadedBody).not.toContain("synthetic_password_abcdefgh");+ for (const name of ["api_key", "password"]) {+ expect(field(name)).not.toMatch(/synthetic|credential|password|abcdefgh/);+ }+ });++ it("removes non-local IP addresses while retaining their surrounding context", async () => {+ await uploadSyntheticBundle();+ expect(uploadedBody).toContain("remote=");+ expect(uploadedBody).not.toContain("203.0.113.29");+ expect(field("remote")).not.toMatch(/\d/);+ });++ it("removes home-directory usernames across supported path formats", async () => {+ await uploadSyntheticBundle();+ expect(uploadedBody).toContain("linux=");+ expect(uploadedBody).toContain("windows=");+ expect(uploadedBody).not.toContain("synthetic-person");+ });++ it("keeps redacting dashed and international phone numbers", async () => {+ await uploadSyntheticBundle();+ expect(uploadedBody).toContain("phones=");+ expect(uploadedBody).not.toContain("234-567-8901");+ expect(uploadedBody).not.toContain("+1-234-567-8901");+ expect(field("phones")).not.toMatch(/\d/);});});