Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover distinct audio segments and preserved search behavior

Louis Beaumont68f26983ff86
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index c5d572e84..738fc0a6e 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.
This does not establish native recording continuity, real authentication,
account-switch race safety, browser integration, isolation or model performance.
-The current app corpus contains 133 git-mined regressions. See
+The current app corpus contains 134 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -1687,3 +1687,17 @@ equivalent missing-object predicate. Compilation/setup failures are infrastructu
errors. The malformed-database error is supplied by a synthetic SQLite trigger,
not an actually corrupt database. This does not establish migration safety,
capture continuity, real corruption recovery, host isolation or model capability.
+
+## Distinct audio segments in search
+
+`app-audio-distinct-segment-search` executes the actual database search methods
+against disposable SQLite data. Six outcomes cover same-chunk/offset/time
+segments, identical words with distinct boundaries, tag joins, deterministic
+bidirectional pagination, ordinary filters and read-only preservation. The
+broken parent fails three outcomes and preserves three; the historical source-only
+fix passes six. Run `bun test evals/coding-agent/calibrate-audio-segments.test.js`
+for broken, correct, equivalent, unused-fix, grouping/order/filter
+mutations and missing-source controls. Hidden fixtures appear only at grading;
+no build-cache links are declared. This does not prove recording continuity,
+HTTP or hybrid-storage integration, concurrent writes, host isolation or model
+performance.
diff --git a/evals/coding-agent/calibrate-audio-segments.test.js b/evals/coding-agent/calibrate-audio-segments.test.js
new file mode 100644
index 000000000..7970afc9f
--- /dev/null
+++ b/evals/coding-agent/calibrate-audio-segments.test.js
@@ -0,0 +1,50 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, writeFileSync, rmSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='app-audio-distinct-segment-search');
+const source = item.oracle_paths[0];
+const git = (...args) => execFileSync('git',args,{cwd:repo,encoding:'utf8',maxBuffer:128*1024*1024});
+const parent=git('show',`${item.base_ref}:${source}`), fixed=git('show',`${item.oracle_ref}:${source}`);
+const fixture=readFileSync(join(import.meta.dir,'graders/audio-segment-search.rs'));
+const hash=x=>createHash('sha256').update(x).digest('hex');
+function change(text,from,to,count=1){expect(text.split(from).length-1).toBe(count);return text.split(from).join(to);}
+test('audio segment grader rejects broken, bypass and preserved-behavior regressions',()=>{
+ const root=mkdtempSync(join(tmpdir(),'audio-segment-calibration-'));
+ try {
+ const archive=join(root,'source.tar');
+ execFileSync('git',['archive','--output',archive,item.base_ref],{cwd:repo,timeout:30000});
+ execFileSync('tar',['-xf',archive,'-C',root],{timeout:30000});
+ rmSync(archive);
+ writeFileSync(join(root,'crates/screenpipe-db/tests/eval_audio_segments.rs'),fixture);
+ const controls=[
+ ['parent',parent,'fail'],['reference',fixed,'pass'],
+ ['equivalent',change(fixed,'GROUP BY audio_transcriptions.id ORDER BY','GROUP BY audio_transcriptions.id, audio_transcriptions.audio_chunk_id ORDER BY'),'pass'],
+ ['unused',parent,'fail'],
+ ['text-grouping',change(fixed,'GROUP BY audio_transcriptions.id ORDER BY','GROUP BY audio_transcriptions.transcription ORDER BY'),'fail'],
+ ['missing-tie-break',change(fixed,', audio_transcriptions.id {order_dir} LIMIT',' LIMIT'),'fail'],
+ ['tag-split',change(fixed,'GROUP BY audio_transcriptions.id ORDER BY','GROUP BY audio_transcriptions.id, tags.id ORDER BY'),'fail'],
+ ['ignore-device',change(fixed,'query_builder = query_builder.bind(dev);','query_builder = query_builder.bind("Synthetic Output");'),'fail'],
+ ['missing',null,'error'],
+ ];
+ for(const [name,text,expected] of controls){
+ if(text===null)rmSync(join(root,source));else writeFileSync(join(root,source),text);
+ if(name==='unused')writeFileSync(join(root,'unused-correct.rs'),fixed);else rmSync(join(root,'unused-correct.rs'),{force:true});
+ // Calibration alone reuses its own target. No evaluated agent runs here.
+ const result=spawnSync('/bin/bash',['-c',item.grader.command],{cwd:root,encoding:'utf8',timeout:item.grader.timeout_seconds*1000,maxBuffer:8*1024*1024});
+ const kind=classifyGraderError(result);const observed=result.error||result.signal||kind?'error':result.status===0?'pass':'fail';
+ if(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS){const dir=resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS);mkdirSync(dir,{recursive:true});writeFileSync(join(dir,`${name}.json`),JSON.stringify({expected,observed,command:item.grader.command,status:result.status,signal:result.signal,error:result.error?.message??null,error_kind:kind,source_sha256:text===null?null:hash(text),fixture_sha256:hash(fixture),stdout:result.stdout,stderr:result.stderr},null,2));}
+ expect(result.error).toBeUndefined();expect(result.signal).toBeNull();expect(observed).toBe(expected);
+ if(expected==='pass')expect(result.stdout).toContain('6 passed; 0 failed');
+ if(expected==='fail'){expect(result.status).toBe(101);expect(result.stdout).toContain('test result: FAILED.');}
+ if(name==='parent')expect(result.stdout).toContain('3 passed; 3 failed');
+ if(name==='missing')expect(kind).toBe('rust_compile_error');
+ }
+ }finally{rmSync(root,{recursive:true,force:true});}
+},1500000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 0354f0427..81c6947d5 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5514,6 +5514,42 @@
],
"command": "cargo test --locked -p screenpipe-redact --test eval_missing_schema -- --test-threads=1"
}
+ },
+ {
+ "id": "app-audio-distinct-segment-search",
+ "name": "Preserve distinct speech segments in audio search",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "acf844b85625df212436a7db8c68b370def720fe"
+ },
+ "base_ref": "6e9f48bfd69751428ec5c4b10483689845058e4c",
+ "oracle_ref": "acf844b85625df212436a7db8c68b370def720fe",
+ "oracle_paths": [
+ "crates/screenpipe-db/src/db/search.rs"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "crates/screenpipe-db/src/db/search.rs"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "audio",
+ "search",
+ "pagination"
+ ],
+ "prompt": "Audio search collapses distinct recorded speech segments when they share an audio chunk, offset and capture timestamp. Repair the real database search path so every segment remains independently visible, including identical words in separate chunks with distinct segment start/end times. Preserve transcription text, segment boundaries and stored rows. Joining multiple chunk tags must not repeat result rows or tags. With capture timestamp, chunk and offset tied, ascending order follows insertion order and descending order reverses it; offset/limit pagination must visit each segment once. Preserve ordinary distinct-offset records, exact all-requested-tag filtering, absent-tag and unknown-device refusal, combined device and minimum/maximum text length filters, unmatched text search, date bounds, zero limit and exhausted offsets. Reads must not rewrite or delete stored transcriptions. Use disposable synthetic SQLite databases, never user recordings or live services. No particular SQL grouping, helper or query spelling is required. The scope is database search, not recording admission, HTTP delivery, hybrid storage or speech recognition quality.",
+ "agent_timeout_seconds": 1800,
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/audio-segment-search.rs",
+ "destination_path": "crates/screenpipe-db/tests/eval_audio_segments.rs"
+ }
+ ],
+ "command": "cargo test --offline --locked -p screenpipe-db --test eval_audio_segments -- --test-threads=1"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/audio-segment-search.rs b/evals/coding-agent/graders/audio-segment-search.rs
new file mode 100644
index 000000000..0f29f25ae
--- /dev/null
+++ b/evals/coding-agent/graders/audio-segment-search.rs
@@ -0,0 +1,101 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+use chrono::{DateTime, Utc};
+use screenpipe_db::{AudioDevice, AudioResult, DatabaseManager, DeviceType, Order, TagContentType};
+
+struct Fixture {
+ _dir: tempfile::TempDir,
+ db: DatabaseManager,
+ snapshot: Vec<(i64, String, i64, f64, f64)>,
+}
+async fn snapshot(db: &DatabaseManager) -> Vec<(i64, String, i64, f64, f64)> {
+ sqlx::query_as("SELECT id,transcription,offset_index,start_time,end_time FROM audio_transcriptions ORDER BY id")
+ .fetch_all(&db.pool).await.unwrap()
+}
+async fn fixture(shared_offset: bool, tagged: bool, same_text: bool) -> Fixture {
+ let dir = tempfile::tempdir().unwrap();
+ let db = DatabaseManager::new(dir.path().join("synthetic.sqlite").to_str().unwrap(), Default::default()).await.unwrap();
+ let chunk = db.insert_audio_chunk("synthetic-speech.mp4", None).await.unwrap();
+ let device = AudioDevice { name: "Synthetic Output".into(), device_type: DeviceType::Output };
+ let captured = DateTime::parse_from_rfc3339("2026-01-02T12:00:00Z").unwrap().with_timezone(&Utc);
+ for (i, text) in ["first note", "second statement", "third phrase"].iter().enumerate() {
+ let segment_chunk = if same_text { db.insert_audio_chunk(&format!("synthetic-{i}.mp4"), None).await.unwrap() } else { chunk };
+ db.insert_audio_transcription(segment_chunk, text,
+ if shared_offset { 0 } else { i as i64 }, "synthetic", &device, None,
+ Some(i as f64 * 2.0), Some(i as f64 * 2.0 + 2.0), Some(captured)).await.unwrap();
+ if same_text { assert_eq!(db.update_audio_transcription(segment_chunk, "repeated words").await.unwrap(), 1); }
+ }
+ if tagged { db.add_tags(chunk, TagContentType::Audio, vec!["juniper".into(), "review".into()]).await.unwrap(); }
+ let before = snapshot(&db).await;
+ assert_eq!(before.len(), 3, "synthetic setup must retain three distinct stored rows");
+ Fixture { _dir: dir, db, snapshot: before }
+}
+async fn read(f: &Fixture, order: Order, limit: u32, offset: u32) -> Vec<AudioResult> {
+ f.db.search_audio_ordered("", limit, offset, None, None, None, None, None, None, None, None, &[], order).await.unwrap()
+}
+fn segments(rows: &[AudioResult]) -> Vec<String> { rows.iter().map(|r| r.transcription.clone()).collect() }
+async fn unchanged(f: &Fixture) { assert_eq!(snapshot(&f.db).await, f.snapshot); }
+
+#[tokio::test]
+async fn distinct_segments_with_shared_chunk_offset_and_timestamp_survive() {
+ let f = fixture(true, false, false).await;
+ let rows = read(&f, Order::Ascending, 20, 0).await;
+ assert_eq!(segments(&rows), vec!["first note", "second statement", "third phrase"]);
+ for (i, row) in rows.iter().enumerate() {
+ assert_eq!(row.start_time, Some(i as f64 * 2.0));
+ assert_eq!(row.end_time, Some(i as f64 * 2.0 + 2.0));
+ assert!(row.tags.is_empty());
+ }
+ unchanged(&f).await;
+}
+#[tokio::test]
+async fn tag_joins_do_not_duplicate_tags_or_merge_speech() {
+ let f = fixture(true, true, false).await;
+ let rows = read(&f, Order::Ascending, 20, 0).await;
+ assert_eq!(rows.len(), 3);
+ for row in rows { let mut tags = row.tags; tags.sort(); assert_eq!(tags, vec!["juniper", "review"]); }
+ unchanged(&f).await;
+}
+#[tokio::test]
+async fn tied_segments_paginate_once_in_both_directions() {
+ let f = fixture(true, true, false).await;
+ for (order, expected) in [(Order::Ascending, vec!["first note", "second statement", "third phrase"]),
+ (Order::Descending, vec!["third phrase", "second statement", "first note"])] {
+ let mut actual = Vec::new();
+ for offset in 0..4 { actual.extend(segments(&read(&f, order, 1, offset).await)); }
+ assert_eq!(actual, expected);
+ }
+ unchanged(&f).await;
+}
+#[tokio::test]
+async fn identical_words_in_distinct_segments_are_not_deduplicated() {
+ let f = fixture(true, false, true).await;
+ let rows = read(&f, Order::Ascending, 20, 0).await;
+ assert_eq!(segments(&rows), vec!["repeated words"; 3]);
+ assert_eq!(rows.iter().map(|r| r.start_time).collect::<Vec<_>>(), vec![Some(0.0), Some(2.0), Some(4.0)]);
+ unchanged(&f).await;
+}
+#[tokio::test]
+async fn ordinary_offsets_preserve_filters_and_metadata() {
+ let f = fixture(false, true, false).await;
+ let tags = vec!["juniper".to_string(), "review".to_string()];
+ let rows = f.db.search_audio_ordered("", 20, 0, None, None, Some(11), Some(13), None, None,
+ Some("Synthetic Output"), None, &tags, Order::Ascending).await.unwrap();
+ assert_eq!(segments(&rows), vec!["third phrase"]);
+ assert_eq!(rows[0].start_time, Some(4.0));
+ let missing = vec!["juniper".to_string(), "missing".to_string()];
+ assert!(f.db.search_audio_ordered("", 20, 0, None, None, None, None, None, None, None, None, &missing, Order::Ascending).await.unwrap().is_empty());
+ assert!(f.db.search_audio_ordered("", 20, 0, None, None, None, None, None, None, Some("Unknown Device"), None, &[], Order::Ascending).await.unwrap().is_empty());
+ unchanged(&f).await;
+}
+#[tokio::test]
+async fn empty_results_and_read_only_behavior_are_preserved() {
+ let f = fixture(false, false, false).await;
+ assert!(read(&f, Order::Ascending, 0, 0).await.is_empty());
+ assert!(read(&f, Order::Descending, 10, 10).await.is_empty());
+ assert!(f.db.search_audio_ordered("absentword", 20, 0, None, None, None, None, None, None, None, None, &[], Order::Ascending).await.unwrap().is_empty());
+ let future = DateTime::parse_from_rfc3339("2026-02-01T00:00:00Z").unwrap().with_timezone(&Utc);
+ assert!(f.db.search_audio_ordered("", 20, 0, Some(future), None, None, None, None, None, None, None, &[], Order::Ascending).await.unwrap().is_empty());
+ assert_eq!(segments(&read(&f, Order::Ascending, 1, 1).await), vec!["second statement"]);
+ unchanged(&f).await;
+}