YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover pipe write decisions at the registered tool hook
louis4866e7428533
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex c82530768..f6eaab684 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 77 git-mined regressions. See+The current app corpus contains 78 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -875,3 +875,13 @@ without requiring a particular request flag. Missing source remains a setuperror. Hidden fixtures and dependencies are materialized only for grading.These are corpus checks, not model trials, settings persistence, native/liveprovider acceptance or proof of execution isolation.++## Pipe write permission hook++`app-pipe-write-permission-hook` exercises the actual registered extension hook+with synthetic permissions and tool events. Eighteen outcomes include outside+file writes, multi-operand shell writes, expansions and opaque scripts, plus+permitted writes and reads. No commands are executed. This does not establish+complete shell analysis, native enforcement, symlink safety or agent isolation.+Run `bun test evals/coding-agent/calibrate-pipe-write-hook.test.js` for historical,+current, equivalent-implementation, unused-correct and mutation controls.diff --git a/evals/coding-agent/calibrate-pipe-write-hook.test.js b/evals/coding-agent/calibrate-pipe-write-hook.test.jsnew file mode 100644index 000000000..cdba4bf79--- /dev/null+++ b/evals/coding-agent/calibrate-pipe-write-hook.test.js@@ -0,0 +1,55 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';++const repo = resolve(import.meta.dir, '../..');+const root = mkdtempSync(join(tmpdir(), 'pipe-hook-calibration-'));+const c = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-pipe-write-permission-hook');+const file = c.oracle_paths[0];+afterAll(() => rmSync(root, { recursive: true, force: true }));+function grade(name, ref, mutate = source => source) {+ const dir = join(root, name);+ mkdirSync(dirname(join(dir, file)), { recursive: true });+ const source = execFileSync('git', ['show', `${ref}:${file}`], { cwd: repo, encoding: 'utf8' });+ writeFileSync(join(dir, file), mutate(source));+ copyFileSync(join(import.meta.dir, c.grader.fixtures[0].local_path), join(dir, 'eval-pipe-write-hook.test.ts'));+ const result = spawnSync(process.execPath, ['test', 'eval-pipe-write-hook.test.ts'], {+ cwd: dir, encoding: 'utf8', timeout: 20000, env: { PATH: process.env.PATH, CI: 'true', TZ: 'UTC' },+ });+ if (process.env.EVAL_CALIBRATION_OUTPUT) {+ const out = resolve(process.env.EVAL_CALIBRATION_OUTPUT);+ mkdirSync(out, { recursive: true });+ writeFileSync(join(out, name + '.stdout'), result.stdout ?? '');+ writeFileSync(join(out, name + '.stderr'), result.stderr ?? '');+ writeFileSync(join(out, name + '.json'), JSON.stringify({ status: result.status, signal: result.signal, error: result.error?.message ?? null }));+ }+ expect(result.error).toBeUndefined();+ expect(result.signal).toBeNull();+ return result;+}+function pass(result) { expect(result.status).toBe(0); expect(result.stderr).toContain('18 pass'); }+function fail(result, name) {+ expect(result.status).toBe(1);+ expect(result.stderr).toContain('(fail) ' + name);+ expect(result.stderr).toMatch(/expect\(received\)/);+ expect(result.stderr).not.toContain('Cannot find');+}+function replace(source, from, to) { expect(source).toContain(from); return source.replace(from, to); }+const hook = 'pi.on("tool_call", async (event: any) => {';+test('broken parent fails intended hook outcomes and preserves nearby behavior', () => {+ const result = grade('parent', c.base_ref);+ fail(result, 'built-in write outside');+ expect(result.stderr).toContain('10 pass');+ expect(result.stderr).toContain('8 fail');+});+test('historical reference passes', () => pass(grade('reference', c.oracle_ref)));+test('current contract passes', () => pass(grade('current', process.env.EVAL_CURRENT_REF || 'b7493b57550a7dc6128cc820caf10eaa8f480775')));+test('renamed private checks remain equivalent', () => pass(grade('equivalent', c.oracle_ref, s => s.replaceAll('checkFilesystemWrite', 'inspectShellWrite').replaceAll('checkFileToolWrite', 'inspectFileWrite'))));+test('unused correct guard cannot repair the broken hook', () => fail(grade('unused-correct', c.base_ref, s => s + '\nfunction unusedCorrectGuard() { return { block: true }; }\n'), 'built-in write outside'));+test('deny-all shortcut loses preserved writes', () => fail(grade('deny-all', c.oracle_ref, s => replace(s, hook, hook + '\nreturn { block: true };')), 'relative file write'));+test('allow-all shortcut loses write enforcement', () => fail(grade('allow-all', c.oracle_ref, s => replace(s, hook, hook + '\nreturn;')), 'built-in write outside'));+test('blocking read tools loses preserved access', () => fail(grade('block-read', c.oracle_ref, s => replace(s, hook, hook + '\nif (event.tool === "read") return { block: true };')), 'read outside remains available'));diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex b01028f0e..a12011cd1 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -3188,6 +3188,41 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-capacity-plan-picker.config.mjs"}+ },+ {+ "id": "app-pipe-write-permission-hook",+ "name": "Enforce pipe write boundaries at the registered tool hook",+ "source": {+ "kind": "git_regression",+ "fix_commit": "c630f18d186b62ab826f3956e7963a42ce9f9909"+ },+ "base_ref": "47b59ce096d37843b4b04f7401e2ec246ca2e5e3",+ "oracle_ref": "c630f18d186b62ab826f3956e7963a42ce9f9909",+ "oracle_paths": [+ "crates/screenpipe-core/assets/extensions/screenpipe-permissions.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "permissions",+ "pipe-tools"+ ],+ "prompt": "A configured pipe can write outside its permitted directory through the extension tool-call hook. Enforce its configured write boundary for built-in write and edit tools, including both tool/name event fields and file_path/path targets, traversal and missing targets. For shell calls, reject an outside target after a permitted first rm or touch operand, runtime-expanded redirect destinations, opaque Python scripts, outside writes inside a nested shell and outside writes after a permitted compound-command segment. Preserve relative file writes, absolute in-directory edits, reads outside the directory, in-directory shell redirects, multiple permitted write targets, literal dollar characters in single-quoted in-directory filenames, read-only shell commands and interpreter version queries. Any implementation achieving these outcomes is acceptable; no helper name, parser architecture or error wording is required. Exercise the actual registered extension hook using synthetic configuration and event inputs. Do not execute the supplied commands or access real user files. This case grades hook decisions for these command forms, not complete shell analysis, native filesystem enforcement, symlink handling or agent isolation.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/pipe-write-hook.fixture.ts",+ "destination_path": "eval-pipe-write-hook.test.ts"+ }+ ],+ "command": "bun test eval-pipe-write-hook.test.ts"+ },+ "trigger_paths": [+ "crates/screenpipe-core/assets/extensions/screenpipe-permissions.ts"+ ]}]}diff --git a/evals/coding-agent/graders/pipe-write-hook.fixture.ts b/evals/coding-agent/graders/pipe-write-hook.fixture.tsnew file mode 100644index 000000000..6fb196cf5--- /dev/null+++ b/evals/coding-agent/graders/pipe-write-hook.fixture.ts@@ -0,0 +1,51 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { pathToFileURL } from 'node:url';+const originalCwd = process.cwd();+const source = join(originalCwd, 'crates/screenpipe-core/assets/extensions/screenpipe-permissions.ts');+const root = mkdtempSync(join(tmpdir(), 'pipe-write-hook-'));+const outside = join(root, '../outside-synthetic.txt');+writeFileSync(join(root, '.screenpipe-permissions.json'), JSON.stringify({ pipe_name: 'report', pipe_dir: root, allow_rules: [], deny_rules: [], use_default_allowlist: false }));+process.chdir(root);+afterAll(() => { process.chdir(originalCwd); rmSync(root, { recursive: true, force: true }); });+const { default: extension } = await import(pathToFileURL(source).href);++const handlers = new Map<string, (event: unknown) => Promise<any>>();+extension({ on: (name: string, handler: (event: unknown) => Promise<any>) => handlers.set(name, handler) } as any);+const call = async (tool: string, input: unknown, alternate = false) => {+ const handler = handlers.get('tool_call');+ expect(handler).toBeFunction();+ return handler!({ [alternate ? 'name' : 'tool']: tool, input });+};+const denied: [string, string, unknown, boolean?][] = [+ ['built-in write outside', 'write', { file_path: outside, content: 'synthetic' }],+ ['alternate edit event outside', 'edit', { path: outside }, true],+ ['traversal outside', 'write', { path: '../outside.txt' }],+ ['missing write target', 'write', {}],+ ['second rm target outside', 'bash', { command: `rm -f ./inside.txt ${outside}` }],+ ['second touch target outside', 'bash', { command: `touch ./inside.txt ${outside}` }],+ ['expanded redirect', 'bash', { command: 'echo synthetic > "$HOME/outside.txt"' }],+ ['opaque interpreter', 'bash', { command: 'python3 task.py' }],+ ['nested shell outside', 'bash', { command: `sh -c "rm -f ${outside}"` }],+ ['compound outside', 'bash', { command: `echo synthetic > ./inside.txt && rm -f ${outside}` }],+];+for (const [name, tool, input, alternate] of denied) {+ test(name, async () => { expect((await call(tool, input, alternate))?.block).toBe(true); });+}+const allowed: [string, string, unknown][] = [+ ['relative file write', 'write', { path: './report.txt', content: 'synthetic' }],+ ['absolute in-root edit', 'edit', { file_path: root + '/report.txt' }],+ ['read outside remains available', 'read', { file_path: outside }],+ ['shell redirect inside', 'bash', { command: 'echo synthetic > ./report.txt' }],+ ['multiple permitted targets', 'bash', { command: 'rm -f ./one.txt ./two.txt' }],+ ['literal dollar within single quotes', 'bash', { command: "echo synthetic > './$HOME.txt'" }],+ ['read-only shell', 'bash', { command: 'cat /synthetic/documents/read.txt' }],+ ['interpreter version query', 'bash', { command: 'python3 --version' }],+];+for (const [name, tool, input] of allowed) {+ test(name, async () => { expect((await call(tool, input))?.block).not.toBe(true); });+}