YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(activity-summary): balance audio context across requested range (#5322)
Akshai Padmanabhan698b5f87b9a1
diff --git a/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.ts b/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.tsindex 9e0a2ee15..c00cc79b8 100644--- a/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.ts+++ b/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.ts@@ -53,6 +53,57 @@ describe("meeting-context image notes", () => {});});+describe("meeting-context transcript fallback", () => {+ it("samples both early and late fragments when only eight fit", () => {+ const meeting: MeetingRecord = {+ id: 42,+ meeting_start: "2026-06-04T15:00:00.000Z",+ meeting_end: "2026-06-04T15:30:00.000Z",+ meeting_app: "zoom",+ title: "Design review",+ attendees: null,+ note: null,+ detection_source: "manual",+ created_at: "2026-06-04T15:00:00.000Z",+ };+ const topTranscriptions = Array.from({ length: 20 }, (_, index) => ({+ transcription: `fragment-${String(index).padStart(2, "0")}`,+ speaker: "Alice",+ device: "mic",+ timestamp: new Date(Date.UTC(2026, 5, 4, 15, index)).toISOString(),+ }));+ const context: MeetingContext = {+ activity: {+ apps: [],+ windows: [],+ audio_summary: {+ segment_count: topTranscriptions.length,+ speakers: [],+ top_transcriptions: topTranscriptions,+ },+ total_frames: 0,+ time_range: {+ start: meeting.meeting_start,+ end: meeting.meeting_end,+ },+ },+ clipboardCount: 0,+ ok: true,+ };++ const prompt = buildEnrichedSummarizePrompt({+ meeting,+ context,+ transcript: [],+ });+ const sampledFragments = prompt.match(/fragment-\d{2}/g) ?? [];++ expect(sampledFragments).toHaveLength(8);+ expect(sampledFragments[0]).toBe("fragment-00");+ expect(sampledFragments[sampledFragments.length - 1]).toBe("fragment-19");+ });+});+function chunk(overrides: Partial<MeetingAudioChunk>): MeetingAudioChunk {return {audioChunkId: 1,diff --git a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts b/apps/screenpipe-app-tauri/lib/utils/meeting-context.tsindex de064fd0d..c1c167ff1 100644--- a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts+++ b/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts@@ -352,6 +352,18 @@ export function pathFromUrl(url: string): string {}}+function evenlySpacedItems<T>(items: readonly T[], limit: number): T[] {+ if (items.length === 0 || limit <= 0) return [];+ if (items.length <= limit) return [...items];+ if (limit === 1) return [items[0]];++ const lastIndex = items.length - 1;+ return Array.from({ length: limit }, (_, index) => {+ const sourceIndex = Math.round((index * lastIndex) / (limit - 1));+ return items[sourceIndex];+ });+}+// ─── Prompt builder ──────────────────────────────────────────────────────interface SummarizeInput {@@ -443,7 +455,10 @@ export function buildEnrichedSummarizePrompt({!transcript?.length &&a.audio_summary.top_transcriptions.length > 0) {- const lines = a.audio_summary.top_transcriptions.slice(0, 8).map((t) => {+ const chronological = [...a.audio_summary.top_transcriptions].sort(+ (left, right) => timestampMs(left.timestamp) - timestampMs(right.timestamp),+ );+ const lines = evenlySpacedItems(chronological, 8).map((t) => {const ts = formatTimeShort(t.timestamp);const txt = t.transcription.replace(/\s+/g, " ").trim().slice(0, 240);const sp = t.speaker && t.speaker !== "unknown" ? `[${t.speaker}] ` : "";diff --git a/crates/screenpipe-engine/src/routes/activity_summary.rs b/crates/screenpipe-engine/src/routes/activity_summary.rsindex 8293e0702..d4653b330 100644--- a/crates/screenpipe-engine/src/routes/activity_summary.rs+++ b/crates/screenpipe-engine/src/routes/activity_summary.rs@@ -34,6 +34,7 @@ use screenpipe_db::DatabaseManager;/// by the per-app / per-window `minutes` SQL and the whole-range/// `total_active_minutes` so the three definitions never drift apart.const IDLE_CAP_SECS: i64 = 300;+const MAX_TOP_TRANSCRIPTIONS: u32 = 20;// ---------- query ----------@@ -156,7 +157,7 @@ pub struct SpeakerSummary {pub struct AudioSummary {pub segment_count: i64,pub speakers: Vec<SpeakerSummary>,- /// Top transcriptions with actual text (sorted by length, most substantial first)+ /// Substantial transcriptions sampled across the requested time range.pub top_transcriptions: Vec<AudioSegment>,}@@ -544,20 +545,10 @@ async fn collect_summary_core(GROUP BY at.speaker_id ORDER BY 2 DESC LIMIT 10");- // Top transcriptions by length — the AI summary prompt uses these as- // "notable quotes." Full transcript is fetched separately via /search.- let audio_transcripts_query = format!(- "SELECT at.transcription, \- COALESCE(s.name, 'Unknown') as speaker, \- at.device, \- at.timestamp \- FROM audio_transcriptions at \- LEFT JOIN speakers s ON at.speaker_id = s.id \- WHERE at.timestamp BETWEEN '{start}' AND '{end}' \- AND TRIM(at.transcription) != '' \- AND LENGTH(at.transcription) > 5 \- ORDER BY LENGTH(at.transcription) DESC LIMIT 20"- );+ // Keep notable quotes representative of the whole requested range. A+ // newest-first or global length sort lets one recent meeting consume the+ // context for a weekly summary.+ let audio_transcripts_query = balanced_audio_query(start, end, "", MAX_TOP_TRANSCRIPTIONS);// Cap at 50 paths — a 1000-file workspace would be noise anyway.let edited_files_query = format!(@@ -869,16 +860,7 @@ async fn load_snippets(}).unwrap_or_default();- let audio_query = format!(- "SELECT at.transcription, COALESCE(s.name, 'Unknown') AS speaker, at.timestamp \- FROM audio_transcriptions at \- LEFT JOIN speakers s ON at.speaker_id = s.id \- WHERE at.timestamp BETWEEN '{start}' AND '{end}'{audio_text_filter} \- AND TRIM(at.transcription) != '' \- AND LENGTH(at.transcription) > 5 \- ORDER BY at.timestamp DESC \- LIMIT {audio_limit}"- );+ let audio_query = balanced_audio_query(start, end, &audio_text_filter, audio_limit);let screen_candidates: Vec<&KeyText> = key_texts.iter()@@ -972,6 +954,49 @@ fn evenly_spaced_indices(len: usize, limit: usize) -> Vec<usize> {indices}+/// Select useful audio from evenly sized wall-clock buckets across the+/// requested interval. Rows are selected in rank rounds across occupied+/// buckets, so the whole range is represented before later ranks backfill any+/// unused capacity. The selected rows are returned chronologically.+fn balanced_audio_query(start: &str, end: &str, extra_filter: &str, limit: u32) -> String {+ let limit = limit.max(1);+ let last_bucket = limit - 1;++ format!(+ "WITH bucketed AS ( \+ SELECT at.transcription, \+ COALESCE(s.name, 'Unknown') AS speaker, \+ at.device, \+ at.timestamp, \+ MIN({last_bucket}, CAST( \+ ((JULIANDAY(at.timestamp) - JULIANDAY('{start}')) \+ / NULLIF(JULIANDAY('{end}') - JULIANDAY('{start}'), 0)) \+ * {limit} AS INTEGER \+ )) AS time_bucket \+ FROM audio_transcriptions at \+ LEFT JOIN speakers s ON at.speaker_id = s.id \+ WHERE at.timestamp BETWEEN '{start}' AND '{end}'{extra_filter} \+ AND TRIM(at.transcription) != '' \+ AND LENGTH(at.transcription) > 5 \+ ), ranked AS ( \+ SELECT transcription, speaker, device, timestamp, time_bucket, \+ ROW_NUMBER() OVER ( \+ PARTITION BY time_bucket \+ ORDER BY LENGTH(transcription) DESC, timestamp DESC \+ ) AS bucket_rank \+ FROM bucketed \+ ), selected AS ( \+ SELECT transcription, speaker, device, timestamp \+ FROM ranked \+ ORDER BY bucket_rank ASC, time_bucket ASC, timestamp ASC \+ LIMIT {limit} \+ ) \+ SELECT transcription, speaker, device, timestamp \+ FROM selected \+ ORDER BY timestamp ASC"+ )+}+// ---------- status + guidance ----------fn compute_data_status(@@ -1897,6 +1922,145 @@ mod db_tests {assert_eq!(core.audio_summary.top_transcriptions.len(), 2);}+ #[tokio::test]+ async fn weekly_audio_context_is_not_consumed_by_the_last_few_minutes() {+ let (db, _d) = fresh_db().await;+ db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")+ .await+ .unwrap();+ db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")+ .await+ .unwrap();++ // One useful segment per day, followed by enough longer segments in+ // the final minutes to crowd every older row out of the old global+ // length sort and newest-first snippet query.+ for day in 1..=7 {+ db.execute_raw_sql(&format!(+ "INSERT INTO audio_transcriptions \+ (audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \+ VALUES (1, {day}, '2026-06-0{day} 10:00:00', \+ 'weekly audio day {day} project update with decisions and next steps', 'mic', 1)"+ ))+ .await+ .unwrap();+ }+ for second in 0..24 {+ db.execute_raw_sql(&format!(+ "INSERT INTO audio_transcriptions \+ (audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \+ VALUES (1, {}, '2026-06-07 23:59:{second:02}', \+ 'recent burst {second:02} with substantially longer detailed discussion that must not replace the rest of the requested week', \+ 'mic', 1)",+ 100 + second+ ))+ .await+ .unwrap();+ }++ let start = "2026-06-01 00:00:00";+ let end = "2026-06-08 00:00:00";+ let core = collect_summary_core(&db, &query(None), start, end).await;+ assert_eq!(+ core.audio_summary.top_transcriptions.len(),+ MAX_TOP_TRANSCRIPTIONS as usize,+ "unused slots should be backfilled after every occupied period is represented"+ );+ let top_days = core+ .audio_summary+ .top_transcriptions+ .iter()+ .map(|segment| segment.timestamp.get(..10).unwrap_or(""))+ .collect::<std::collections::HashSet<_>>();++ assert!(+ top_days.contains("2026-06-01"),+ "top transcriptions lost early-week audio: {top_days:?}"+ );+ assert!(+ top_days.contains("2026-06-07"),+ "top transcriptions should still include recent audio: {top_days:?}"+ );+ assert_eq!(+ top_days.len(),+ 7,+ "weekly audio should cover the full range, got days {top_days:?}"+ );+ assert!(+ core.audio_summary+ .top_transcriptions+ .windows(2)+ .all(|pair| pair[0].timestamp <= pair[1].timestamp),+ "selected transcriptions should be returned chronologically"+ );++ let mut snippet_query = query(None);+ snippet_query.max_snippets = 12;+ let snippets = load_snippets(&db, &snippet_query, &[], start, end)+ .await+ .expect("load weekly snippets");+ let audio_days = snippets+ .iter()+ .filter(|snippet| snippet.source == "audio")+ .map(|snippet| snippet.timestamp.get(..10).unwrap_or(""))+ .collect::<std::collections::HashSet<_>>();++ assert!(+ audio_days.contains("2026-06-01"),+ "audio snippets lost early-week context: {audio_days:?}"+ );+ assert!(+ audio_days.contains("2026-06-07"),+ "audio snippets should still include recent context: {audio_days:?}"+ );+ assert!(+ audio_days.len() >= 5,+ "audio snippets should span the week, got days {audio_days:?}"+ );+ }++ #[tokio::test]+ async fn audio_context_backfills_capacity_from_one_dense_period() {+ let (db, _d) = fresh_db().await;+ db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")+ .await+ .unwrap();+ db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")+ .await+ .unwrap();++ for second in 0..24 {+ db.execute_raw_sql(&format!(+ "INSERT INTO audio_transcriptions \+ (audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \+ VALUES (1, {second}, '2026-06-07 23:59:{second:02}', \+ 'dense meeting transcript {second:02} with enough detail for summary context', \+ 'mic', 1)"+ ))+ .await+ .unwrap();+ }++ let core = collect_summary_core(+ &db,+ &query(None),+ "2026-06-01 00:00:00",+ "2026-06-08 00:00:00",+ )+ .await;++ assert_eq!(+ core.audio_summary.top_transcriptions.len(),+ MAX_TOP_TRANSCRIPTIONS as usize,+ "a single occupied bucket should backfill all available slots"+ );+ assert!(core+ .audio_summary+ .top_transcriptions+ .iter()+ .all(|segment| segment.timestamp.starts_with("2026-06-07 23:59")));+ }+#[tokio::test]async fn key_texts_from_accessibility_element() {let (db, _d) = fresh_db().await;