Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover duplicate shared audio inventories

Louis Beaumontfdf871c46d8c
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 7f7a58a4b..a8eee1fe4 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -2017,3 +2017,19 @@ blanket-denial, no-op-script and missing-source controls. Hidden fixtures and
third-party dependencies appear only when grading starts. This is synthetic
HTTP evidence, not a real browser/DNS exploit test, remote network validation,
all header-parser ambiguities, model performance or enforced host isolation.
+
+## Unique shared audio inventory
+
+`app-shared-inventory-duplicate-devices` executes the actual audio inventory cache
+with synthetic enumeration. Seven outcomes cover first-seen deduplication by
+exact name and direction, cache hits and topology refresh, case/whitespace and
+direction distinctions, empty/single inventories, TTL/error retry, concurrent
+readers and changes during enumeration. The broken parent fails two duplicate
+inventory outcomes and preserves five neighbors; the reference passes seven.
+
+Run `bun test evals/coding-agent/calibrate-audio-inventory.test.js` for nine
+calibration controls, including an equivalent vector implementation, name-only
+merging, sorting, raw-result bypass, cache bypass, unused correct code and lost
+topology events. Each native command retains the 180-second bound. Hidden
+fixtures appear only at grading; this case uses no build-cache links. These are
+synthetic cache tests, not hardware, HTTP, capture/pause, migration or model trials.
diff --git a/evals/coding-agent/calibrate-audio-inventory.test.js b/evals/coding-agent/calibrate-audio-inventory.test.js
new file mode 100644
index 000000000..e61a2ec98
--- /dev/null
+++ b/evals/coding-agent/calibrate-audio-inventory.test.js
@@ -0,0 +1,41 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { join, resolve } from 'node:path';
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-shared-inventory-duplicate-devices');
+const receipts = process.env.EVAL_CALIBRATION_RESULTS_DIR;
+if (receipts) mkdirSync(receipts, { recursive: true });
+const cwd = mkdtempSync(join(receipts || tmpdir(), 'audio-inventory-calibration-'));
+const source = 'crates/screenpipe-audio/src/core/device.rs';
+const fixture = readFileSync(join(import.meta.dir, 'graders/audio-inventory.fixture.rs'), 'utf8');
+const fixed = execFileSync('git', ['show', `${item.oracle_ref}:${source}`], { cwd: repo, encoding: 'utf8' });
+const dedup = ' let mut seen = HashSet::with_capacity(devices.len());\n devices.retain(|device| seen.insert(device.clone()));';
+afterAll(() => { if (!receipts) rmSync(cwd, { recursive: true, force: true }); });
+function replace(text, from, to) { expect(text.split(from)).toHaveLength(2); return text.replace(from, to); }
+function grade(name, ref, mutate = text => text, unusedCorrect = false) {
+ const archive = execFileSync('git', ['archive', ref, 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'crates'], { cwd: repo, maxBuffer: 64 * 1024 * 1024 });
+ execFileSync('tar', ['-xf', '-', '-C', cwd], { input: archive });
+ writeFileSync(join(cwd, source), mutate(readFileSync(join(cwd, source), 'utf8')) + '\n' + fixture);
+ const unused = join(cwd, 'crates/screenpipe-audio/src/core/unused_correct.rs');
+ if (unusedCorrect) writeFileSync(unused, fixed);
+ const r = spawnSync('cargo', ['test', '--locked', '-p', 'screenpipe-audio', '--lib', 'eval_audio_inventory', '--', '--test-threads=1'], { cwd, encoding: 'utf8', timeout: 180_000 });
+ if (unusedCorrect) rmSync(unused);
+ if (receipts) writeFileSync(join(receipts, name + '.json'), JSON.stringify({ ref, status: r.status, signal: r.signal, error: r.error?.message, stdout: r.stdout, stderr: r.stderr, failure_kind: classifyGraderError(r) }, null, 2));
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.stdout).toContain('running 7 tests'); expect(classifyGraderError(r)).toBeNull();
+ return r;
+}
+function fails(r) { expect(r.status).toBe(101); expect(r.stdout).toContain('test result: FAILED.'); }
+test('parent fails duplicate inventories and preserves five neighbors', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stdout).toContain('5 passed; 2 failed'); }, 200_000);
+test('reference passes every outcome', () => { const r = grade('reference', item.oracle_ref); expect(r.status).toBe(0); expect(r.stdout).toContain('7 passed'); }, 200_000);
+test('equivalent stable vector strategy is accepted', () => expect(grade('equivalent', item.oracle_ref, t => replace(t, dedup, ' let mut seen = Vec::new();\n devices.retain(|device| { if seen.contains(device) { false } else { seen.push(device.clone()); true } });')).status).toBe(0), 200_000);
+test('name-only deduplication is rejected', () => fails(grade('name-only', item.oracle_ref, t => replace(t, 'seen.insert(device.clone())', 'seen.insert(device.name.clone())'))), 200_000);
+test('sorting the inventory is rejected', () => fails(grade('sorting', item.oracle_ref, t => replace(t, dedup, dedup + '\n devices.sort_by(|a, b| a.name.cmp(&b.name));'))), 200_000);
+test('returning raw duplicates despite a correct cache is rejected', () => fails(grade('raw-return', item.oracle_ref, t => replace(replace(t, ' let mut devices = enumerate().await?;', ' let mut devices = enumerate().await?;\n let raw_devices = devices.clone();'), ' Ok(devices)\n}', ' Ok(raw_devices)\n}'))), 200_000);
+test('bypassing the cache is rejected', () => fails(grade('no-cache', item.oracle_ref, t => replace(t, 'if c.topology_generation == topology_generation && c.fetched_at.elapsed() < DEVICE_CACHE_TTL', 'if false && c.topology_generation == topology_generation && c.fetched_at.elapsed() < DEVICE_CACHE_TTL'))), 200_000);
+test('unused correct code cannot hide broken inventory', () => fails(grade('unused-correct', item.base_ref, t => t, true)), 200_000);
+test('consuming events during enumeration is rejected', () => fails(grade('lost-event', item.oracle_ref, t => replace(t, ' topology_generation,\n });', ' topology_generation: generation(),\n });'))), 200_000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 4f5fec701..7a9ff1d4b 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6739,6 +6739,45 @@
],
"command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 SCREENPIPE_MCP_TELEMETRY_DISABLED=1 DO_NOT_TRACK=1 SCREENPIPE_API_URL=http://127.0.0.1:1 node node_modules/vitest/vitest.mjs run src/eval-browser-boundary.test.ts"
}
+ },
+ {
+ "id": "app-shared-inventory-duplicate-devices",
+ "name": "Return a unique audio device inventory without losing cache behavior",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "c6c6382768caf87a83758f65b13e5f6bf53ffb5a"
+ },
+ "base_ref": "5ce7edfcda145fa1bb074574925b290a1356bf2c",
+ "oracle_ref": "c6c6382768caf87a83758f65b13e5f6bf53ffb5a",
+ "oracle_paths": [
+ "crates/screenpipe-audio/src/core/device.rs"
+ ],
+ "trigger_paths": [
+ "crates/screenpipe-audio/src/core/device.rs"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "audio",
+ "device-inventory"
+ ],
+ "prompt": "The shared audio inventory can repeat the same device when platform enumeration returns duplicate entries. Make every fresh or cached inventory contain each exact device name and input/output direction once, preserving first-seen order. Same-named inputs and outputs remain distinct, as do names differing in case or whitespace. Empty and single-device inventories remain valid. Preserve the existing cache contract: unexpired same-generation reads reuse the inventory, concurrent readers share one enumeration, TTL expiry or a topology-generation change refreshes it, refresh replaces removed devices, enumeration errors propagate and remain retryable, and a topology change during enumeration forces another refresh on the next read. Preserve the existing AudioDevice and list_audio_devices_with_cache interfaces so synthetic enumeration can exercise the real cache function. No particular deduplication algorithm is required. Use synthetic devices only; do not access recording hardware, customer data or external services. This task does not require changing capture, pause, migration or HTTP behavior.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/audio-inventory.mjs",
+ "destination_path": ".eval-audio-inventory/audio-inventory.mjs"
+ },
+ {
+ "local_path": "graders/audio-inventory.fixture.rs",
+ "destination_path": ".eval-audio-inventory/audio-inventory.fixture.rs"
+ }
+ ],
+ "command": "node .eval-audio-inventory/audio-inventory.mjs && cargo test --locked -p screenpipe-audio --lib eval_audio_inventory -- --test-threads=1"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/audio-inventory.fixture.rs b/evals/coding-agent/graders/audio-inventory.fixture.rs
new file mode 100644
index 000000000..bb8784950
--- /dev/null
+++ b/evals/coding-agent/graders/audio-inventory.fixture.rs
@@ -0,0 +1,189 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+#[cfg(test)]
+mod eval_audio_inventory {
+ use super::*;
+ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+ fn device(name: &str, output: bool) -> AudioDevice {
+ AudioDevice::new(
+ name.into(),
+ if output {
+ DeviceType::Output
+ } else {
+ DeviceType::Input
+ },
+ )
+ }
+ async fn inventory(
+ cache: &AsyncMutex<Option<CachedDevices>>,
+ generation: &AtomicU64,
+ calls: &AtomicUsize,
+ devices: Vec<AudioDevice>,
+ ) -> Vec<AudioDevice> {
+ list_audio_devices_with_cache(
+ cache,
+ || generation.load(Ordering::Acquire),
+ || async {
+ calls.fetch_add(1, Ordering::Relaxed);
+ tokio::task::yield_now().await;
+ Ok(devices)
+ },
+ )
+ .await
+ .unwrap()
+ }
+ #[tokio::test]
+ async fn unique_inventory_and_cached_result() {
+ let cache = AsyncMutex::new(None);
+ let gen = AtomicU64::new(0);
+ let calls = AtomicUsize::new(0);
+ let a = device("Zebra headset", false);
+ let b = device("Desk mic", false);
+ let c = device("Zebra headset", true);
+ let expected = vec![a.clone(), b.clone(), c.clone()];
+ let actual = inventory(&cache, &gen, &calls, vec![a.clone(), b, a, c.clone(), c]).await;
+ assert_eq!(
+ actual, expected,
+ "duplicate name+direction entries must appear once in first-seen order"
+ );
+ assert_eq!(
+ inventory(&cache, &gen, &calls, vec![]).await,
+ expected,
+ "cache hit must retain unique inventory"
+ );
+ assert_eq!(
+ calls.load(Ordering::Relaxed),
+ 1,
+ "cache hit must not enumerate again"
+ );
+ }
+ #[tokio::test]
+ async fn refreshed_duplicates_and_removed_devices() {
+ let cache = AsyncMutex::new(None);
+ let gen = AtomicU64::new(0);
+ let calls = AtomicUsize::new(0);
+ inventory(&cache, &gen, &calls, vec![device("Old mic", false)]).await;
+ gen.fetch_add(1, Ordering::Release);
+ let a = device("New mic", false);
+ let b = device("New mic", true);
+ assert_eq!(
+ inventory(
+ &cache,
+ &gen,
+ &calls,
+ vec![b.clone(), a.clone(), b.clone(), a.clone()]
+ )
+ .await,
+ vec![b, a],
+ "topology refresh must replace old inventory and deduplicate new entries"
+ );
+ gen.fetch_add(1, Ordering::Release);
+ assert!(inventory(&cache, &gen, &calls, vec![]).await.is_empty());
+ assert!(
+ inventory(&cache, &gen, &calls, vec![device("Unobserved", false)])
+ .await
+ .is_empty()
+ );
+ assert_eq!(calls.load(Ordering::Relaxed), 3);
+ }
+ #[tokio::test]
+ async fn exact_names_directions_and_order_preserved() {
+ let cache = AsyncMutex::new(None);
+ let gen = AtomicU64::new(0);
+ let calls = AtomicUsize::new(0);
+ let expected = vec![
+ device("zeta", true),
+ device("Alpha", false),
+ device("alpha", false),
+ device("Alpha ", false),
+ device("Alpha", true),
+ ];
+ assert_eq!(
+ inventory(&cache, &gen, &calls, expected.clone()).await,
+ expected,
+ "distinct exact names and directions must not be merged or sorted"
+ );
+ }
+ #[tokio::test]
+ async fn empty_and_single_inventory_preserved() {
+ let cache = AsyncMutex::new(None);
+ let gen = AtomicU64::new(0);
+ let calls = AtomicUsize::new(0);
+ assert!(inventory(&cache, &gen, &calls, vec![]).await.is_empty());
+ gen.fetch_add(1, Ordering::Release);
+ assert_eq!(
+ inventory(&cache, &gen, &calls, vec![device("Single", false)]).await,
+ vec![device("Single", false)]
+ );
+ }
+ #[tokio::test]
+ async fn ttl_expiry_and_failed_refresh_retry() {
+ let cache = AsyncMutex::new(Some(CachedDevices {
+ devices: vec![device("Stale", false)],
+ fetched_at: Instant::now() - DEVICE_CACHE_TTL,
+ topology_generation: 0,
+ }));
+ let gen = AtomicU64::new(0);
+ let calls = AtomicUsize::new(0);
+ assert_eq!(
+ inventory(&cache, &gen, &calls, vec![device("Fresh", false)]).await,
+ vec![device("Fresh", false)]
+ );
+ gen.fetch_add(1, Ordering::Release);
+ let failed = list_audio_devices_with_cache(
+ &cache,
+ || gen.load(Ordering::Acquire),
+ || async { Err(anyhow!("synthetic enumeration failure")) },
+ )
+ .await;
+ assert!(failed.is_err(), "enumeration errors must propagate");
+ assert_eq!(
+ inventory(&cache, &gen, &calls, vec![device("Retry", false)]).await,
+ vec![device("Retry", false)],
+ "failed refresh must remain retryable"
+ );
+ assert_eq!(calls.load(Ordering::Relaxed), 2);
+ }
+ #[tokio::test]
+ async fn concurrent_readers_share_one_enumeration() {
+ let cache = AsyncMutex::new(None);
+ let gen = AtomicU64::new(0);
+ let calls = AtomicUsize::new(0);
+ let expected = vec![device("Shared", false)];
+ let results = futures::future::join_all(
+ (0..8).map(|_| inventory(&cache, &gen, &calls, expected.clone())),
+ )
+ .await;
+ assert!(results.iter().all(|v| v == &expected));
+ assert_eq!(
+ calls.load(Ordering::Relaxed),
+ 1,
+ "concurrent readers must coalesce"
+ );
+ }
+ #[tokio::test]
+ async fn change_during_enumeration_requires_following_refresh() {
+ let cache = AsyncMutex::new(None);
+ let gen = AtomicU64::new(0);
+ let calls = AtomicUsize::new(0);
+ list_audio_devices_with_cache(
+ &cache,
+ || gen.load(Ordering::Acquire),
+ || async {
+ gen.fetch_add(1, Ordering::Release);
+ Ok(vec![device("Old", false)])
+ },
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ inventory(&cache, &gen, &calls, vec![device("New", false)]).await,
+ vec![device("New", false)]
+ );
+ assert_eq!(
+ calls.load(Ordering::Relaxed),
+ 1,
+ "event during enumeration cannot be consumed early"
+ );
+ }
+}
diff --git a/evals/coding-agent/graders/audio-inventory.mjs b/evals/coding-agent/graders/audio-inventory.mjs
new file mode 100644
index 000000000..2804a6d14
--- /dev/null
+++ b/evals/coding-agent/graders/audio-inventory.mjs
@@ -0,0 +1,4 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { appendFileSync, readFileSync } from 'node:fs';
+appendFileSync('crates/screenpipe-audio/src/core/device.rs', '\n' + readFileSync(new URL('./audio-inventory.fixture.rs', import.meta.url), 'utf8'));