Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(activity-summary): balance audio context across requested range (#5322)

Akshai Padmanabhan698b5f87b9a1
diff --git a/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.ts b/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.ts
index 9e0a2ee15..c00cc79b8 100644
--- a/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.ts
+++ b/apps/screenpipe-app-tauri/lib/utils/meeting-context.test.ts
@@ -53,6 +53,57 @@ describe("meeting-context image notes", () => {
});
});
+describe("meeting-context transcript fallback", () => {
+ it("samples both early and late fragments when only eight fit", () => {
+ const meeting: MeetingRecord = {
+ id: 42,
+ meeting_start: "2026-06-04T15:00:00.000Z",
+ meeting_end: "2026-06-04T15:30:00.000Z",
+ meeting_app: "zoom",
+ title: "Design review",
+ attendees: null,
+ note: null,
+ detection_source: "manual",
+ created_at: "2026-06-04T15:00:00.000Z",
+ };
+ const topTranscriptions = Array.from({ length: 20 }, (_, index) => ({
+ transcription: `fragment-${String(index).padStart(2, "0")}`,
+ speaker: "Alice",
+ device: "mic",
+ timestamp: new Date(Date.UTC(2026, 5, 4, 15, index)).toISOString(),
+ }));
+ const context: MeetingContext = {
+ activity: {
+ apps: [],
+ windows: [],
+ audio_summary: {
+ segment_count: topTranscriptions.length,
+ speakers: [],
+ top_transcriptions: topTranscriptions,
+ },
+ total_frames: 0,
+ time_range: {
+ start: meeting.meeting_start,
+ end: meeting.meeting_end,
+ },
+ },
+ clipboardCount: 0,
+ ok: true,
+ };
+
+ const prompt = buildEnrichedSummarizePrompt({
+ meeting,
+ context,
+ transcript: [],
+ });
+ const sampledFragments = prompt.match(/fragment-\d{2}/g) ?? [];
+
+ expect(sampledFragments).toHaveLength(8);
+ expect(sampledFragments[0]).toBe("fragment-00");
+ expect(sampledFragments[sampledFragments.length - 1]).toBe("fragment-19");
+ });
+});
+
function chunk(overrides: Partial<MeetingAudioChunk>): MeetingAudioChunk {
return {
audioChunkId: 1,
diff --git a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts b/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts
index de064fd0d..c1c167ff1 100644
--- a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts
+++ b/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts
@@ -352,6 +352,18 @@ export function pathFromUrl(url: string): string {
}
}
+function evenlySpacedItems<T>(items: readonly T[], limit: number): T[] {
+ if (items.length === 0 || limit <= 0) return [];
+ if (items.length <= limit) return [...items];
+ if (limit === 1) return [items[0]];
+
+ const lastIndex = items.length - 1;
+ return Array.from({ length: limit }, (_, index) => {
+ const sourceIndex = Math.round((index * lastIndex) / (limit - 1));
+ return items[sourceIndex];
+ });
+}
+
// ─── Prompt builder ──────────────────────────────────────────────────────
interface SummarizeInput {
@@ -443,7 +455,10 @@ export function buildEnrichedSummarizePrompt({
!transcript?.length &&
a.audio_summary.top_transcriptions.length > 0
) {
- const lines = a.audio_summary.top_transcriptions.slice(0, 8).map((t) => {
+ const chronological = [...a.audio_summary.top_transcriptions].sort(
+ (left, right) => timestampMs(left.timestamp) - timestampMs(right.timestamp),
+ );
+ const lines = evenlySpacedItems(chronological, 8).map((t) => {
const ts = formatTimeShort(t.timestamp);
const txt = t.transcription.replace(/\s+/g, " ").trim().slice(0, 240);
const sp = t.speaker && t.speaker !== "unknown" ? `[${t.speaker}] ` : "";
diff --git a/crates/screenpipe-engine/src/routes/activity_summary.rs b/crates/screenpipe-engine/src/routes/activity_summary.rs
index 8293e0702..d4653b330 100644
--- a/crates/screenpipe-engine/src/routes/activity_summary.rs
+++ b/crates/screenpipe-engine/src/routes/activity_summary.rs
@@ -34,6 +34,7 @@ use screenpipe_db::DatabaseManager;
/// by the per-app / per-window `minutes` SQL and the whole-range
/// `total_active_minutes` so the three definitions never drift apart.
const IDLE_CAP_SECS: i64 = 300;
+const MAX_TOP_TRANSCRIPTIONS: u32 = 20;
// ---------- query ----------
@@ -156,7 +157,7 @@ pub struct SpeakerSummary {
pub struct AudioSummary {
pub segment_count: i64,
pub speakers: Vec<SpeakerSummary>,
- /// Top transcriptions with actual text (sorted by length, most substantial first)
+ /// Substantial transcriptions sampled across the requested time range.
pub top_transcriptions: Vec<AudioSegment>,
}
@@ -544,20 +545,10 @@ async fn collect_summary_core(
GROUP BY at.speaker_id ORDER BY 2 DESC LIMIT 10"
);
- // Top transcriptions by length — the AI summary prompt uses these as
- // "notable quotes." Full transcript is fetched separately via /search.
- let audio_transcripts_query = format!(
- "SELECT at.transcription, \
- COALESCE(s.name, 'Unknown') as speaker, \
- at.device, \
- at.timestamp \
- FROM audio_transcriptions at \
- LEFT JOIN speakers s ON at.speaker_id = s.id \
- WHERE at.timestamp BETWEEN '{start}' AND '{end}' \
- AND TRIM(at.transcription) != '' \
- AND LENGTH(at.transcription) > 5 \
- ORDER BY LENGTH(at.transcription) DESC LIMIT 20"
- );
+ // Keep notable quotes representative of the whole requested range. A
+ // newest-first or global length sort lets one recent meeting consume the
+ // context for a weekly summary.
+ let audio_transcripts_query = balanced_audio_query(start, end, "", MAX_TOP_TRANSCRIPTIONS);
// Cap at 50 paths — a 1000-file workspace would be noise anyway.
let edited_files_query = format!(
@@ -869,16 +860,7 @@ async fn load_snippets(
})
.unwrap_or_default();
- let audio_query = format!(
- "SELECT at.transcription, COALESCE(s.name, 'Unknown') AS speaker, at.timestamp \
- FROM audio_transcriptions at \
- LEFT JOIN speakers s ON at.speaker_id = s.id \
- WHERE at.timestamp BETWEEN '{start}' AND '{end}'{audio_text_filter} \
- AND TRIM(at.transcription) != '' \
- AND LENGTH(at.transcription) > 5 \
- ORDER BY at.timestamp DESC \
- LIMIT {audio_limit}"
- );
+ let audio_query = balanced_audio_query(start, end, &audio_text_filter, audio_limit);
let screen_candidates: Vec<&KeyText> = key_texts
.iter()
@@ -972,6 +954,49 @@ fn evenly_spaced_indices(len: usize, limit: usize) -> Vec<usize> {
indices
}
+/// Select useful audio from evenly sized wall-clock buckets across the
+/// requested interval. Rows are selected in rank rounds across occupied
+/// buckets, so the whole range is represented before later ranks backfill any
+/// unused capacity. The selected rows are returned chronologically.
+fn balanced_audio_query(start: &str, end: &str, extra_filter: &str, limit: u32) -> String {
+ let limit = limit.max(1);
+ let last_bucket = limit - 1;
+
+ format!(
+ "WITH bucketed AS ( \
+ SELECT at.transcription, \
+ COALESCE(s.name, 'Unknown') AS speaker, \
+ at.device, \
+ at.timestamp, \
+ MIN({last_bucket}, CAST( \
+ ((JULIANDAY(at.timestamp) - JULIANDAY('{start}')) \
+ / NULLIF(JULIANDAY('{end}') - JULIANDAY('{start}'), 0)) \
+ * {limit} AS INTEGER \
+ )) AS time_bucket \
+ FROM audio_transcriptions at \
+ LEFT JOIN speakers s ON at.speaker_id = s.id \
+ WHERE at.timestamp BETWEEN '{start}' AND '{end}'{extra_filter} \
+ AND TRIM(at.transcription) != '' \
+ AND LENGTH(at.transcription) > 5 \
+ ), ranked AS ( \
+ SELECT transcription, speaker, device, timestamp, time_bucket, \
+ ROW_NUMBER() OVER ( \
+ PARTITION BY time_bucket \
+ ORDER BY LENGTH(transcription) DESC, timestamp DESC \
+ ) AS bucket_rank \
+ FROM bucketed \
+ ), selected AS ( \
+ SELECT transcription, speaker, device, timestamp \
+ FROM ranked \
+ ORDER BY bucket_rank ASC, time_bucket ASC, timestamp ASC \
+ LIMIT {limit} \
+ ) \
+ SELECT transcription, speaker, device, timestamp \
+ FROM selected \
+ ORDER BY timestamp ASC"
+ )
+}
+
// ---------- status + guidance ----------
fn compute_data_status(
@@ -1897,6 +1922,145 @@ mod db_tests {
assert_eq!(core.audio_summary.top_transcriptions.len(), 2);
}
+ #[tokio::test]
+ async fn weekly_audio_context_is_not_consumed_by_the_last_few_minutes() {
+ let (db, _d) = fresh_db().await;
+ db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
+ .await
+ .unwrap();
+ db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
+ .await
+ .unwrap();
+
+ // One useful segment per day, followed by enough longer segments in
+ // the final minutes to crowd every older row out of the old global
+ // length sort and newest-first snippet query.
+ for day in 1..=7 {
+ db.execute_raw_sql(&format!(
+ "INSERT INTO audio_transcriptions \
+ (audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \
+ VALUES (1, {day}, '2026-06-0{day} 10:00:00', \
+ 'weekly audio day {day} project update with decisions and next steps', 'mic', 1)"
+ ))
+ .await
+ .unwrap();
+ }
+ for second in 0..24 {
+ db.execute_raw_sql(&format!(
+ "INSERT INTO audio_transcriptions \
+ (audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \
+ VALUES (1, {}, '2026-06-07 23:59:{second:02}', \
+ 'recent burst {second:02} with substantially longer detailed discussion that must not replace the rest of the requested week', \
+ 'mic', 1)",
+ 100 + second
+ ))
+ .await
+ .unwrap();
+ }
+
+ let start = "2026-06-01 00:00:00";
+ let end = "2026-06-08 00:00:00";
+ let core = collect_summary_core(&db, &query(None), start, end).await;
+ assert_eq!(
+ core.audio_summary.top_transcriptions.len(),
+ MAX_TOP_TRANSCRIPTIONS as usize,
+ "unused slots should be backfilled after every occupied period is represented"
+ );
+ let top_days = core
+ .audio_summary
+ .top_transcriptions
+ .iter()
+ .map(|segment| segment.timestamp.get(..10).unwrap_or(""))
+ .collect::<std::collections::HashSet<_>>();
+
+ assert!(
+ top_days.contains("2026-06-01"),
+ "top transcriptions lost early-week audio: {top_days:?}"
+ );
+ assert!(
+ top_days.contains("2026-06-07"),
+ "top transcriptions should still include recent audio: {top_days:?}"
+ );
+ assert_eq!(
+ top_days.len(),
+ 7,
+ "weekly audio should cover the full range, got days {top_days:?}"
+ );
+ assert!(
+ core.audio_summary
+ .top_transcriptions
+ .windows(2)
+ .all(|pair| pair[0].timestamp <= pair[1].timestamp),
+ "selected transcriptions should be returned chronologically"
+ );
+
+ let mut snippet_query = query(None);
+ snippet_query.max_snippets = 12;
+ let snippets = load_snippets(&db, &snippet_query, &[], start, end)
+ .await
+ .expect("load weekly snippets");
+ let audio_days = snippets
+ .iter()
+ .filter(|snippet| snippet.source == "audio")
+ .map(|snippet| snippet.timestamp.get(..10).unwrap_or(""))
+ .collect::<std::collections::HashSet<_>>();
+
+ assert!(
+ audio_days.contains("2026-06-01"),
+ "audio snippets lost early-week context: {audio_days:?}"
+ );
+ assert!(
+ audio_days.contains("2026-06-07"),
+ "audio snippets should still include recent context: {audio_days:?}"
+ );
+ assert!(
+ audio_days.len() >= 5,
+ "audio snippets should span the week, got days {audio_days:?}"
+ );
+ }
+
+ #[tokio::test]
+ async fn audio_context_backfills_capacity_from_one_dense_period() {
+ let (db, _d) = fresh_db().await;
+ db.execute_raw_sql("INSERT INTO audio_chunks (id, file_path) VALUES (1, 'test.wav')")
+ .await
+ .unwrap();
+ db.execute_raw_sql("INSERT INTO speakers (id, name) VALUES (1, 'Alice')")
+ .await
+ .unwrap();
+
+ for second in 0..24 {
+ db.execute_raw_sql(&format!(
+ "INSERT INTO audio_transcriptions \
+ (audio_chunk_id, offset_index, timestamp, transcription, device, speaker_id) \
+ VALUES (1, {second}, '2026-06-07 23:59:{second:02}', \
+ 'dense meeting transcript {second:02} with enough detail for summary context', \
+ 'mic', 1)"
+ ))
+ .await
+ .unwrap();
+ }
+
+ let core = collect_summary_core(
+ &db,
+ &query(None),
+ "2026-06-01 00:00:00",
+ "2026-06-08 00:00:00",
+ )
+ .await;
+
+ assert_eq!(
+ core.audio_summary.top_transcriptions.len(),
+ MAX_TOP_TRANSCRIPTIONS as usize,
+ "a single occupied bucket should backfill all available slots"
+ );
+ assert!(core
+ .audio_summary
+ .top_transcriptions
+ .iter()
+ .all(|segment| segment.timestamp.starts_with("2026-06-07 23:59")));
+ }
+
#[tokio::test]
async fn key_texts_from_accessibility_element() {
let (db, _d) = fresh_db().await;