Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover synchronous audio stream drop cleanup

Louis Beaumont3930b10b8f3e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index f2b8abf61..68e8d322f 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,18 @@
# Coding-agent regression evals
+`app-audio-drop-control-dispatch` executes the actual AudioStream destructor
+with synthetic streams and no capture hardware. Three Drop outcomes distinguish
+the broken parent from the reference while two checks preserve live-stream and
+explicit-stop behavior. Run `bun test evals/coding-agent/calibrate-audio-drop.test.js`
+for seven calibration controls, including equivalent ordering, disconnected
+correct code, missing dispatch and missing disconnect state. The historical
+workspace uses its pinned Rust toolchain and locked native dependencies; ONNX
+Runtime must be available through the normal dependency setup. Setup failures
+are infrastructure errors, not intended baseline failures. Fixtures appear only
+at grading. Native worker completion, deferred device teardown timing, capture
+admission, durable recording, execution isolation and model trials are outside
+this case.
+
`app-telemetry-full-ax-wire` compiles the complete telemetry records module and
runs its public batch encoder and decoder with synthetic input. Twelve outcomes
cover retained accessibility trees, empty nodes, nested and unknown properties,
diff --git a/evals/coding-agent/calibrate-audio-drop.test.js b/evals/coding-agent/calibrate-audio-drop.test.js
new file mode 100644
index 000000000..9e022f7a4
--- /dev/null
+++ b/evals/coding-agent/calibrate-audio-drop.test.js
@@ -0,0 +1,46 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { join, resolve } from 'node:path';
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-audio-drop-control-dispatch');
+const receipts = process.env.EVAL_CALIBRATION_RESULTS_DIR;
+if (receipts) mkdirSync(receipts, { recursive: true });
+const cwd = mkdtempSync(join(receipts || tmpdir(), 'audio-drop-calibration-'));
+const source = 'crates/screenpipe-audio/src/core/stream.rs';
+const fixed = execFileSync('git', ['show', `${item.oracle_ref}:${source}`], { cwd: repo, encoding: 'utf8' });
+const fixture = readFileSync(join(import.meta.dir, 'graders/audio-drop.fixture.rs'), 'utf8');
+const dropBody = ` let _ = self
+ .stream_control
+ .send(StreamControl::stop_without_wait(StopMode::Immediate));
+ self.is_disconnected.store(true, Ordering::Relaxed);`;
+const flag = ' self.is_disconnected.store(true, Ordering::Relaxed);';
+afterAll(() => { if (!receipts) rmSync(cwd, { recursive: true, force: true }); });
+function replace(text, from, to) { expect(text.split(from)).toHaveLength(2); return text.replace(from, to); }
+function grade(name, ref, mutate = text => text, unusedCorrect = false) {
+ const archive = execFileSync('git', ['archive', ref, 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'crates'], { cwd: repo, maxBuffer: 64 * 1024 * 1024 });
+ execFileSync('tar', ['-xf', '-', '-C', cwd], { input: archive });
+ const path = join(cwd, source), body = mutate(readFileSync(path, 'utf8'));
+ writeFileSync(path, body + '\n' + fixture);
+ const unused = join(cwd, 'crates/screenpipe-audio/src/core/unused_correct.rs');
+ if (unusedCorrect) writeFileSync(unused, fixed);
+ const env = { ...process.env };
+ if (process.env.SCREENPIPE_EVAL_CARGO_TARGET_DIR) env.CARGO_TARGET_DIR = process.env.SCREENPIPE_EVAL_CARGO_TARGET_DIR;
+ const r = spawnSync('cargo', ['test', '--locked', '-p', 'screenpipe-audio', '--lib', 'eval_audio_drop', '--', '--test-threads=1'], { cwd, env, encoding: 'utf8', timeout: 180_000 });
+ if (unusedCorrect) rmSync(unused);
+ if (receipts) writeFileSync(join(receipts, name + '.json'), JSON.stringify({ ref, status: r.status, signal: r.signal, error: r.error?.message, stdout: r.stdout, stderr: r.stderr, failure_kind: classifyGraderError(r) }, null, 2));
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.stdout).toContain('running 5 tests'); expect(classifyGraderError(r)).toBeNull();
+ return r;
+}
+function fails(r, count) { expect(r.status).toBe(101); expect(r.stdout).toContain(`${count} failed`); }
+test('parent fails three Drop outcomes and preserves two neighbors', () => { const r = grade('parent', item.base_ref); fails(r, 3); expect(r.stdout).toContain('2 passed'); }, 200_000);
+test('reference passes all five outcomes', () => { const r = grade('reference', item.oracle_ref); expect(r.status).toBe(0); expect(r.stdout).toContain('5 passed'); }, 200_000);
+test('equivalent ordering is accepted', () => expect(grade('equivalent', item.oracle_ref, t => replace(t, dropBody, flag + '\n' + dropBody.replace('\n' + flag, ''))).status).toBe(0), 200_000);
+test('missing disconnect state is rejected', () => fails(grade('missing-disconnect', item.oracle_ref, t => replace(t, dropBody, dropBody.replace('\n' + flag, ''))), 2), 200_000);
+test('missing control dispatch is rejected', () => fails(grade('missing-send', item.oracle_ref, t => replace(t, dropBody, flag)), 1), 200_000);
+test('unused correct code cannot hide broken Drop', () => fails(grade('unused-correct', item.oracle_ref, t => replace(t, dropBody, ''), true), 3), 200_000);
+test('preserved explicit stop remains required', () => fails(grade('broken-stop', item.oracle_ref, t => replace(t, ' pub async fn stop(&self) -> Result<()> {\n' + flag, ' pub async fn stop(&self) -> Result<()> {')), 1), 200_000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 6eb1da37d..4c2a81eb5 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6658,6 +6658,45 @@
],
"command": "cargo test --offline --manifest-path .eval-telemetry-ax/Cargo.toml --test outcomes -- --nocapture"
}
+ },
+ {
+ "id": "app-audio-drop-control-dispatch",
+ "name": "Dispatch audio shutdown before dropping the stream",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "cd1fe138819e573b89e975283e4d19c8307748ed"
+ },
+ "base_ref": "e53b0f0938e47d7d161e4e50a088c43ac04a402c",
+ "oracle_ref": "cd1fe138819e573b89e975283e4d19c8307748ed",
+ "oracle_paths": [
+ "crates/screenpipe-audio/src/core/stream.rs"
+ ],
+ "trigger_paths": [
+ "crates/screenpipe-audio/src/core/stream.rs"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "audio",
+ "resource-lifecycle"
+ ],
+ "prompt": "Dropping an AudioStream outside an asynchronous runtime can leave its shutdown request unsent and shared disconnect state unchanged. Repair the actual stream destructor so dropping a synthetic stream synchronously marks it disconnected and dispatches its stop request before returning, even when no executor is polling. If the control receiver is already closed, dropping must still mark it disconnected without panicking. A live stream must remain connected until stopped or dropped, and explicit stop with the synthetic closed receiver must remain successful and mark it disconnected. Preserve the existing AudioStream interfaces and synthetic constructor. The hidden checks use the existing internal control sender and shared disconnect flag to observe these outcomes; no particular shutdown helper, task implementation or control-message representation is required. Use synthetic streams only. Native worker completion, device teardown timing, capture hardware, durable recording, migration authority and real customer data are outside this task.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/audio-drop.mjs",
+ "destination_path": ".eval-audio-drop/audio-drop.mjs"
+ },
+ {
+ "local_path": "graders/audio-drop.fixture.rs",
+ "destination_path": ".eval-audio-drop/audio-drop.fixture.rs"
+ }
+ ],
+ "command": "node .eval-audio-drop/audio-drop.mjs && cargo test --locked -p screenpipe-audio --lib eval_audio_drop -- --test-threads=1"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/audio-drop.fixture.rs b/evals/coding-agent/graders/audio-drop.fixture.rs
new file mode 100644
index 000000000..7465c828c
--- /dev/null
+++ b/evals/coding-agent/graders/audio-drop.fixture.rs
@@ -0,0 +1,46 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+#[cfg(test)]
+mod eval_audio_drop {
+ use super::*;
+ fn synthetic() -> AudioStream {
+ let device = Arc::new(AudioDevice::new("synthetic eval input".into(), super::super::device::DeviceType::Input));
+ AudioStream::from_sender_for_test(device, 16_000, 1).0
+ }
+ #[test]
+ fn drop_marks_disconnected_without_runtime() {
+ let stream = synthetic();
+ let flag = stream.is_disconnected.clone();
+ assert!(!flag.load(Ordering::Relaxed));
+ drop(stream);
+ assert!(flag.load(Ordering::Relaxed), "drop must synchronously mark disconnected");
+ }
+ #[test]
+ fn drop_delivers_stop_without_runtime() {
+ let mut stream = synthetic();
+ let (tx, rx) = mpsc::channel();
+ stream.stream_control = tx;
+ assert!(rx.try_recv().is_err());
+ drop(stream);
+ assert!(rx.try_recv().is_ok(), "stop must be delivered before drop returns");
+ }
+ #[test]
+ fn closed_control_channel_does_not_prevent_disconnect() {
+ let stream = synthetic();
+ let flag = stream.is_disconnected.clone();
+ assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(stream))).is_ok());
+ assert!(flag.load(Ordering::Relaxed));
+ }
+ #[test]
+ fn living_stream_remains_connected() {
+ let stream = synthetic();
+ assert!(!stream.is_disconnected());
+ assert!(!stream.is_disconnected.load(Ordering::Relaxed));
+ }
+ #[tokio::test]
+ async fn explicit_stop_preserves_synthetic_stream_cleanup() {
+ let mut stream = synthetic();
+ stream.stop().await.expect("explicit stop with closed synthetic receiver remains successful");
+ assert!(stream.is_disconnected());
+ }
+}
diff --git a/evals/coding-agent/graders/audio-drop.mjs b/evals/coding-agent/graders/audio-drop.mjs
new file mode 100644
index 000000000..68e8c54a3
--- /dev/null
+++ b/evals/coding-agent/graders/audio-drop.mjs
@@ -0,0 +1,4 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { appendFileSync, readFileSync } from 'node:fs';
+appendFileSync('crates/screenpipe-audio/src/core/stream.rs', '\n' + readFileSync(new URL('./audio-drop.fixture.rs', import.meta.url), 'utf8'));