YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover duplicate shared audio inventories
Louis Beaumontfdf871c46d8c
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 7f7a58a4b..a8eee1fe4 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -2017,3 +2017,19 @@ blanket-denial, no-op-script and missing-source controls. Hidden fixtures andthird-party dependencies appear only when grading starts. This is syntheticHTTP evidence, not a real browser/DNS exploit test, remote network validation,all header-parser ambiguities, model performance or enforced host isolation.++## Unique shared audio inventory++`app-shared-inventory-duplicate-devices` executes the actual audio inventory cache+with synthetic enumeration. Seven outcomes cover first-seen deduplication by+exact name and direction, cache hits and topology refresh, case/whitespace and+direction distinctions, empty/single inventories, TTL/error retry, concurrent+readers and changes during enumeration. The broken parent fails two duplicate+inventory outcomes and preserves five neighbors; the reference passes seven.++Run `bun test evals/coding-agent/calibrate-audio-inventory.test.js` for nine+calibration controls, including an equivalent vector implementation, name-only+merging, sorting, raw-result bypass, cache bypass, unused correct code and lost+topology events. Each native command retains the 180-second bound. Hidden+fixtures appear only at grading; this case uses no build-cache links. These are+synthetic cache tests, not hardware, HTTP, capture/pause, migration or model trials.diff --git a/evals/coding-agent/calibrate-audio-inventory.test.js b/evals/coding-agent/calibrate-audio-inventory.test.jsnew file mode 100644index 000000000..e61a2ec98--- /dev/null+++ b/evals/coding-agent/calibrate-audio-inventory.test.js@@ -0,0 +1,41 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { join, resolve } from 'node:path';+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { classifyGraderError } from './grader-outcome.mjs';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-shared-inventory-duplicate-devices');+const receipts = process.env.EVAL_CALIBRATION_RESULTS_DIR;+if (receipts) mkdirSync(receipts, { recursive: true });+const cwd = mkdtempSync(join(receipts || tmpdir(), 'audio-inventory-calibration-'));+const source = 'crates/screenpipe-audio/src/core/device.rs';+const fixture = readFileSync(join(import.meta.dir, 'graders/audio-inventory.fixture.rs'), 'utf8');+const fixed = execFileSync('git', ['show', `${item.oracle_ref}:${source}`], { cwd: repo, encoding: 'utf8' });+const dedup = ' let mut seen = HashSet::with_capacity(devices.len());\n devices.retain(|device| seen.insert(device.clone()));';+afterAll(() => { if (!receipts) rmSync(cwd, { recursive: true, force: true }); });+function replace(text, from, to) { expect(text.split(from)).toHaveLength(2); return text.replace(from, to); }+function grade(name, ref, mutate = text => text, unusedCorrect = false) {+ const archive = execFileSync('git', ['archive', ref, 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo', 'crates'], { cwd: repo, maxBuffer: 64 * 1024 * 1024 });+ execFileSync('tar', ['-xf', '-', '-C', cwd], { input: archive });+ writeFileSync(join(cwd, source), mutate(readFileSync(join(cwd, source), 'utf8')) + '\n' + fixture);+ const unused = join(cwd, 'crates/screenpipe-audio/src/core/unused_correct.rs');+ if (unusedCorrect) writeFileSync(unused, fixed);+ const r = spawnSync('cargo', ['test', '--locked', '-p', 'screenpipe-audio', '--lib', 'eval_audio_inventory', '--', '--test-threads=1'], { cwd, encoding: 'utf8', timeout: 180_000 });+ if (unusedCorrect) rmSync(unused);+ if (receipts) writeFileSync(join(receipts, name + '.json'), JSON.stringify({ ref, status: r.status, signal: r.signal, error: r.error?.message, stdout: r.stdout, stderr: r.stderr, failure_kind: classifyGraderError(r) }, null, 2));+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.stdout).toContain('running 7 tests'); expect(classifyGraderError(r)).toBeNull();+ return r;+}+function fails(r) { expect(r.status).toBe(101); expect(r.stdout).toContain('test result: FAILED.'); }+test('parent fails duplicate inventories and preserves five neighbors', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stdout).toContain('5 passed; 2 failed'); }, 200_000);+test('reference passes every outcome', () => { const r = grade('reference', item.oracle_ref); expect(r.status).toBe(0); expect(r.stdout).toContain('7 passed'); }, 200_000);+test('equivalent stable vector strategy is accepted', () => expect(grade('equivalent', item.oracle_ref, t => replace(t, dedup, ' let mut seen = Vec::new();\n devices.retain(|device| { if seen.contains(device) { false } else { seen.push(device.clone()); true } });')).status).toBe(0), 200_000);+test('name-only deduplication is rejected', () => fails(grade('name-only', item.oracle_ref, t => replace(t, 'seen.insert(device.clone())', 'seen.insert(device.name.clone())'))), 200_000);+test('sorting the inventory is rejected', () => fails(grade('sorting', item.oracle_ref, t => replace(t, dedup, dedup + '\n devices.sort_by(|a, b| a.name.cmp(&b.name));'))), 200_000);+test('returning raw duplicates despite a correct cache is rejected', () => fails(grade('raw-return', item.oracle_ref, t => replace(replace(t, ' let mut devices = enumerate().await?;', ' let mut devices = enumerate().await?;\n let raw_devices = devices.clone();'), ' Ok(devices)\n}', ' Ok(raw_devices)\n}'))), 200_000);+test('bypassing the cache is rejected', () => fails(grade('no-cache', item.oracle_ref, t => replace(t, 'if c.topology_generation == topology_generation && c.fetched_at.elapsed() < DEVICE_CACHE_TTL', 'if false && c.topology_generation == topology_generation && c.fetched_at.elapsed() < DEVICE_CACHE_TTL'))), 200_000);+test('unused correct code cannot hide broken inventory', () => fails(grade('unused-correct', item.base_ref, t => t, true)), 200_000);+test('consuming events during enumeration is rejected', () => fails(grade('lost-event', item.oracle_ref, t => replace(t, ' topology_generation,\n });', ' topology_generation: generation(),\n });'))), 200_000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 4f5fec701..7a9ff1d4b 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6739,6 +6739,45 @@],"command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 SCREENPIPE_MCP_TELEMETRY_DISABLED=1 DO_NOT_TRACK=1 SCREENPIPE_API_URL=http://127.0.0.1:1 node node_modules/vitest/vitest.mjs run src/eval-browser-boundary.test.ts"}+ },+ {+ "id": "app-shared-inventory-duplicate-devices",+ "name": "Return a unique audio device inventory without losing cache behavior",+ "source": {+ "kind": "git_regression",+ "fix_commit": "c6c6382768caf87a83758f65b13e5f6bf53ffb5a"+ },+ "base_ref": "5ce7edfcda145fa1bb074574925b290a1356bf2c",+ "oracle_ref": "c6c6382768caf87a83758f65b13e5f6bf53ffb5a",+ "oracle_paths": [+ "crates/screenpipe-audio/src/core/device.rs"+ ],+ "trigger_paths": [+ "crates/screenpipe-audio/src/core/device.rs"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "audio",+ "device-inventory"+ ],+ "prompt": "The shared audio inventory can repeat the same device when platform enumeration returns duplicate entries. Make every fresh or cached inventory contain each exact device name and input/output direction once, preserving first-seen order. Same-named inputs and outputs remain distinct, as do names differing in case or whitespace. Empty and single-device inventories remain valid. Preserve the existing cache contract: unexpired same-generation reads reuse the inventory, concurrent readers share one enumeration, TTL expiry or a topology-generation change refreshes it, refresh replaces removed devices, enumeration errors propagate and remain retryable, and a topology change during enumeration forces another refresh on the next read. Preserve the existing AudioDevice and list_audio_devices_with_cache interfaces so synthetic enumeration can exercise the real cache function. No particular deduplication algorithm is required. Use synthetic devices only; do not access recording hardware, customer data or external services. This task does not require changing capture, pause, migration or HTTP behavior.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 180,+ "fixtures": [+ {+ "local_path": "graders/audio-inventory.mjs",+ "destination_path": ".eval-audio-inventory/audio-inventory.mjs"+ },+ {+ "local_path": "graders/audio-inventory.fixture.rs",+ "destination_path": ".eval-audio-inventory/audio-inventory.fixture.rs"+ }+ ],+ "command": "node .eval-audio-inventory/audio-inventory.mjs && cargo test --locked -p screenpipe-audio --lib eval_audio_inventory -- --test-threads=1"+ }}]}diff --git a/evals/coding-agent/graders/audio-inventory.fixture.rs b/evals/coding-agent/graders/audio-inventory.fixture.rsnew file mode 100644index 000000000..bb8784950--- /dev/null+++ b/evals/coding-agent/graders/audio-inventory.fixture.rs@@ -0,0 +1,189 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+#[cfg(test)]+mod eval_audio_inventory {+ use super::*;+ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};+ fn device(name: &str, output: bool) -> AudioDevice {+ AudioDevice::new(+ name.into(),+ if output {+ DeviceType::Output+ } else {+ DeviceType::Input+ },+ )+ }+ async fn inventory(+ cache: &AsyncMutex<Option<CachedDevices>>,+ generation: &AtomicU64,+ calls: &AtomicUsize,+ devices: Vec<AudioDevice>,+ ) -> Vec<AudioDevice> {+ list_audio_devices_with_cache(+ cache,+ || generation.load(Ordering::Acquire),+ || async {+ calls.fetch_add(1, Ordering::Relaxed);+ tokio::task::yield_now().await;+ Ok(devices)+ },+ )+ .await+ .unwrap()+ }+ #[tokio::test]+ async fn unique_inventory_and_cached_result() {+ let cache = AsyncMutex::new(None);+ let gen = AtomicU64::new(0);+ let calls = AtomicUsize::new(0);+ let a = device("Zebra headset", false);+ let b = device("Desk mic", false);+ let c = device("Zebra headset", true);+ let expected = vec![a.clone(), b.clone(), c.clone()];+ let actual = inventory(&cache, &gen, &calls, vec![a.clone(), b, a, c.clone(), c]).await;+ assert_eq!(+ actual, expected,+ "duplicate name+direction entries must appear once in first-seen order"+ );+ assert_eq!(+ inventory(&cache, &gen, &calls, vec![]).await,+ expected,+ "cache hit must retain unique inventory"+ );+ assert_eq!(+ calls.load(Ordering::Relaxed),+ 1,+ "cache hit must not enumerate again"+ );+ }+ #[tokio::test]+ async fn refreshed_duplicates_and_removed_devices() {+ let cache = AsyncMutex::new(None);+ let gen = AtomicU64::new(0);+ let calls = AtomicUsize::new(0);+ inventory(&cache, &gen, &calls, vec![device("Old mic", false)]).await;+ gen.fetch_add(1, Ordering::Release);+ let a = device("New mic", false);+ let b = device("New mic", true);+ assert_eq!(+ inventory(+ &cache,+ &gen,+ &calls,+ vec![b.clone(), a.clone(), b.clone(), a.clone()]+ )+ .await,+ vec![b, a],+ "topology refresh must replace old inventory and deduplicate new entries"+ );+ gen.fetch_add(1, Ordering::Release);+ assert!(inventory(&cache, &gen, &calls, vec![]).await.is_empty());+ assert!(+ inventory(&cache, &gen, &calls, vec![device("Unobserved", false)])+ .await+ .is_empty()+ );+ assert_eq!(calls.load(Ordering::Relaxed), 3);+ }+ #[tokio::test]+ async fn exact_names_directions_and_order_preserved() {+ let cache = AsyncMutex::new(None);+ let gen = AtomicU64::new(0);+ let calls = AtomicUsize::new(0);+ let expected = vec![+ device("zeta", true),+ device("Alpha", false),+ device("alpha", false),+ device("Alpha ", false),+ device("Alpha", true),+ ];+ assert_eq!(+ inventory(&cache, &gen, &calls, expected.clone()).await,+ expected,+ "distinct exact names and directions must not be merged or sorted"+ );+ }+ #[tokio::test]+ async fn empty_and_single_inventory_preserved() {+ let cache = AsyncMutex::new(None);+ let gen = AtomicU64::new(0);+ let calls = AtomicUsize::new(0);+ assert!(inventory(&cache, &gen, &calls, vec![]).await.is_empty());+ gen.fetch_add(1, Ordering::Release);+ assert_eq!(+ inventory(&cache, &gen, &calls, vec![device("Single", false)]).await,+ vec![device("Single", false)]+ );+ }+ #[tokio::test]+ async fn ttl_expiry_and_failed_refresh_retry() {+ let cache = AsyncMutex::new(Some(CachedDevices {+ devices: vec![device("Stale", false)],+ fetched_at: Instant::now() - DEVICE_CACHE_TTL,+ topology_generation: 0,+ }));+ let gen = AtomicU64::new(0);+ let calls = AtomicUsize::new(0);+ assert_eq!(+ inventory(&cache, &gen, &calls, vec![device("Fresh", false)]).await,+ vec![device("Fresh", false)]+ );+ gen.fetch_add(1, Ordering::Release);+ let failed = list_audio_devices_with_cache(+ &cache,+ || gen.load(Ordering::Acquire),+ || async { Err(anyhow!("synthetic enumeration failure")) },+ )+ .await;+ assert!(failed.is_err(), "enumeration errors must propagate");+ assert_eq!(+ inventory(&cache, &gen, &calls, vec![device("Retry", false)]).await,+ vec![device("Retry", false)],+ "failed refresh must remain retryable"+ );+ assert_eq!(calls.load(Ordering::Relaxed), 2);+ }+ #[tokio::test]+ async fn concurrent_readers_share_one_enumeration() {+ let cache = AsyncMutex::new(None);+ let gen = AtomicU64::new(0);+ let calls = AtomicUsize::new(0);+ let expected = vec![device("Shared", false)];+ let results = futures::future::join_all(+ (0..8).map(|_| inventory(&cache, &gen, &calls, expected.clone())),+ )+ .await;+ assert!(results.iter().all(|v| v == &expected));+ assert_eq!(+ calls.load(Ordering::Relaxed),+ 1,+ "concurrent readers must coalesce"+ );+ }+ #[tokio::test]+ async fn change_during_enumeration_requires_following_refresh() {+ let cache = AsyncMutex::new(None);+ let gen = AtomicU64::new(0);+ let calls = AtomicUsize::new(0);+ list_audio_devices_with_cache(+ &cache,+ || gen.load(Ordering::Acquire),+ || async {+ gen.fetch_add(1, Ordering::Release);+ Ok(vec![device("Old", false)])+ },+ )+ .await+ .unwrap();+ assert_eq!(+ inventory(&cache, &gen, &calls, vec![device("New", false)]).await,+ vec![device("New", false)]+ );+ assert_eq!(+ calls.load(Ordering::Relaxed),+ 1,+ "event during enumeration cannot be consumed early"+ );+ }+}diff --git a/evals/coding-agent/graders/audio-inventory.mjs b/evals/coding-agent/graders/audio-inventory.mjsnew file mode 100644index 000000000..2804a6d14--- /dev/null+++ b/evals/coding-agent/graders/audio-inventory.mjs@@ -0,0 +1,4 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { appendFileSync, readFileSync } from 'node:fs';+appendFileSync('crates/screenpipe-audio/src/core/device.rs', '\n' + readFileSync(new URL('./audio-inventory.fixture.rs', import.meta.url), 'utf8'));