YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(meetings): make speaker N and me transcript lines renameable (#5024)
Divanshue7e47fd8960c
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsxindex d788033a3..45c938830 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx+++ b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx@@ -198,7 +198,7 @@ function groupBySpeaker(chunks: MeetingAudioChunk[]): SpeakerBlock[] {const ts = timestampMs(c.timestamp);if (ts <= 0) continue;const speakerName = c.speakerName || (c.isInput ? "me" : "speaker");- const speakerId = c.isInput ? null : c.speakerId;+ const speakerId = c.speakerId;const last = out[out.length - 1];const sameSpeaker =last &&diff --git a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts b/apps/screenpipe-app-tauri/lib/utils/meeting-context.tsindex be949bd56..de064fd0d 100644--- a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts+++ b/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts@@ -832,8 +832,10 @@ export async function fetchMeetingAudio(out.push({audioChunkId: id,audioFilePath: c.file_path,- speakerId: isInput ? null : c.speaker?.id ?? null,- speakerName: isInput ? "me" : c.speaker?.name ?? "",+ speakerId: c.speaker?.id ?? null,+ // Mic rows show "me" only until someone is actually assigned —+ // hardcoding "me" made input lines look impossible to reassign.+ speakerName: c.speaker?.name?.trim() || (isInput ? "me" : ""),deviceType,isInput,transcription: c.transcription,@@ -906,10 +908,12 @@ async function fetchRoutedMeetingTranscript(? segment.audioChunkId: -segment.id,audioFilePath: segment.audioFilePath ?? "",- speakerId: isInput ? null : segment.speakerId ?? null,- speakerName: isInput- ? "me"- : segment.speakerName?.trim() || "speaker",+ speakerId: segment.speakerId ?? null,+ // The endpoint only returns a name for mic rows once a real speaker+ // is assigned, so an assigned name survives here instead of being+ // masked by a hardcoded "me".+ speakerName:+ segment.speakerName?.trim() || (isInput ? "me" : "speaker"),deviceType,isInput,transcription: segment.transcript,diff --git a/crates/screenpipe-db/src/db/meetings.rs b/crates/screenpipe-db/src/db/meetings.rsindex 29f07bf1e..97f72eab9 100644--- a/crates/screenpipe-db/src/db/meetings.rs+++ b/crates/screenpipe-db/src/db/meetings.rs@@ -902,11 +902,96 @@ impl DatabaseManager {Ok(updated)}+ /// Fill `audio_chunk_id`/`audio_file_path` on live segments from the+ /// `audio_transcriptions` rows the post-meeting mirror wrote for them+ /// (same text, same device direction, ±2s). Live segments carry no chunk+ /// link of their own, and without one the UI can neither rename nor play+ /// a line. Resolving at read time (instead of joining in SQL) keeps the+ /// scan bounded to one pass and works for meetings mirrored before this+ /// existed. Matching in memory mirrors+ /// `mirror_live_meeting_to_audio_transcriptions`, which created the rows.+ async fn resolve_live_segment_chunk_links(+ &self,+ segments: &mut [MeetingTranscriptSegment],+ ) -> Result<(), SqlxError> {+ let mut unresolved: Vec<(usize, i64)> = segments+ .iter()+ .enumerate()+ .filter(|(_, s)| s.source == "live" && s.audio_chunk_id.is_none())+ .filter_map(|(i, s)| {+ let ts = DateTime::parse_from_rfc3339(&s.captured_at).ok()?;+ Some((i, ts.timestamp_millis()))+ })+ .collect();+ if unresolved.is_empty() {+ return Ok(());+ }++ const WINDOW_MS: i64 = 2_000;+ let min_ts = unresolved.iter().map(|(_, ts)| *ts).min().unwrap_or(0);+ let max_ts = unresolved.iter().map(|(_, ts)| *ts).max().unwrap_or(0);+ let bound = |ms: i64| {+ chrono::DateTime::<Utc>::from_timestamp_millis(ms)+ .unwrap_or_default()+ .to_rfc3339()+ };++ let candidate_rows = sqlx::query(+ "SELECT at.transcription, at.timestamp, at.audio_chunk_id, \+ at.is_input_device, ac.file_path \+ FROM audio_transcriptions at \+ JOIN audio_chunks ac ON ac.id = at.audio_chunk_id \+ WHERE at.transcription_engine = 'live' \+ AND julianday(at.timestamp) >= julianday(?1) \+ AND julianday(at.timestamp) <= julianday(?2)",+ )+ .bind(bound(min_ts - WINDOW_MS))+ .bind(bound(max_ts + WINDOW_MS))+ .fetch_all(&self.pool)+ .await?;++ // (is_input, text) → [(ts_ms, chunk_id, file_path)]+ let mut candidates: std::collections::HashMap<(bool, String), Vec<(i64, i64, String)>> =+ std::collections::HashMap::new();+ for row in &candidate_rows {+ let (Ok(text), Ok(ts), Ok(chunk_id)) = (+ row.try_get::<String, _>("transcription"),+ row.try_get::<DateTime<Utc>, _>("timestamp"),+ row.try_get::<i64, _>("audio_chunk_id"),+ ) else {+ continue;+ };+ let is_input = row.try_get::<bool, _>("is_input_device").unwrap_or(true);+ let file_path = row.try_get::<String, _>("file_path").unwrap_or_default();+ candidates+ .entry((is_input, text.trim().to_string()))+ .or_default()+ .push((ts.timestamp_millis(), chunk_id, file_path));+ }++ for (idx, seg_ms) in unresolved.drain(..) {+ let seg = &mut segments[idx];+ let key = (seg.device_type == "input", seg.transcript.trim().to_string());+ let Some(matches) = candidates.get(&key) else {+ continue;+ };+ if let Some((_, chunk_id, file_path)) = matches+ .iter()+ .filter(|(ts, _, _)| (ts - seg_ms).abs() <= WINDOW_MS)+ .min_by_key(|(ts, _, _)| (ts - seg_ms).abs())+ {+ seg.audio_chunk_id = Some(*chunk_id);+ seg.audio_file_path = Some(file_path.clone());+ }+ }+ Ok(())+ }+pub async fn list_meeting_transcript_segments(&self,meeting_id: i64,) -> Result<Vec<MeetingTranscriptSegment>, SqlxError> {- let rows = sqlx::query_as::<_, MeetingTranscriptSegment>(+ let mut rows = sqlx::query_as::<_, MeetingTranscriptSegment>(r#"WITH meeting_window AS (SELECT@@ -933,10 +1018,15 @@ impl DatabaseManager {NULL AS audio_chunk_id,NULL AS audio_file_path,mts.speaker_id AS speaker_id,- -- Prefer the resolved global speaker's name; fall back to the- -- free-text Deepgram label until backfilled / if the speaker is- -- unnamed (NULLIF treats '' as "no name yet").- COALESCE(NULLIF(s.name, ''), mts.speaker_name) AS speaker_name,+ -- Prefer the resolved global speaker's name; output rows fall+ -- back to the free-text Deepgram label until backfilled / if+ -- the speaker is unnamed (NULLIF treats '' as "no name yet").+ -- Input (mic) rows get NULL instead of the raw label so the+ -- client can render "me" until the user assigns someone.+ CASE+ WHEN mts.device_type = 'input' THEN NULLIF(s.name, '')+ ELSE COALESCE(NULLIF(s.name, ''), mts.speaker_name)+ END AS speaker_name,mts.transcript,mts.captured_at,mts.created_at@@ -1012,6 +1102,8 @@ impl DatabaseManager {.bind(meeting_id).fetch_all(&self.pool).await?;++ self.resolve_live_segment_chunk_links(&mut rows).await?;Ok(rows)}diff --git a/crates/screenpipe-db/src/db/speakers.rs b/crates/screenpipe-db/src/db/speakers.rsindex 06ee2a3f8..67bc5cee4 100644--- a/crates/screenpipe-db/src/db/speakers.rs+++ b/crates/screenpipe-db/src/db/speakers.rs@@ -950,18 +950,16 @@ impl DatabaseManager {let (current_speaker_id, target_speaker_id, transcriptions_updated, mut embeddings_moved) = {let mut tx = self.begin_immediate_with_retry().await?;- // 1. Get the current speaker_id for this audio chunk- let current_speaker_id: Option<i64> = sqlx::query_scalar(+ // 1. Get the current speaker_id for this audio chunk. NULL is a+ // legitimate state (mic rows and freshly-mirrored live rows have+ // no speaker until backfill) — only a missing row is an error.+ let current_speaker_id: Option<i64> = sqlx::query_scalar::<_, Option<i64>>("SELECT speaker_id FROM audio_transcriptions WHERE audio_chunk_id = ? LIMIT 1",).bind(audio_chunk_id).fetch_optional(&mut **tx.conn())- .await?;-- let current_speaker_id = match current_speaker_id {- Some(id) => id,- None => return Err(sqlx::Error::RowNotFound),- };+ .await?+ .ok_or(sqlx::Error::RowNotFound)?;// 2. Find or create the target speaker (pick the one with most embeddings// to act as canonical when duplicates exist)@@ -991,14 +989,20 @@ impl DatabaseManager {}};- // Record old assignments for undo- let affected_rows: Vec<(i64, i64)> = sqlx::query_as(+ // Record old assignments for undo. Rows without a speaker can't be+ // represented in the (id, old_speaker_id) undo payload — skip them;+ // undo just leaves those on the new speaker.+ let affected_rows: Vec<(i64, Option<i64>)> = sqlx::query_as("SELECT id, speaker_id FROM audio_transcriptions WHERE audio_chunk_id = ?",).bind(audio_chunk_id).fetch_all(&mut **tx.conn()).await?;- old_assignments.extend(affected_rows);+ old_assignments.extend(+ affected_rows+ .into_iter()+ .filter_map(|(id, speaker)| speaker.map(|s| (id, s))),+ );// 3. Update the transcription's speaker_idlet transcriptions_updated = sqlx::query(@@ -1010,22 +1014,42 @@ impl DatabaseManager {.await?.rows_affected();- // 4. Move one embedding from old speaker to new speaker- let embedding_id: Option<i64> = sqlx::query_scalar(- "SELECT id FROM speaker_embeddings WHERE speaker_id = ? LIMIT 1",+ // Live meeting segments mirrored onto this chunk read their label+ // from meeting_transcript_segments, not audio_transcriptions — sync+ // them (matched by exact text + near-identical timestamp) or the+ // Meeting view keeps showing the old speaker after a rename.+ sqlx::query(+ "UPDATE meeting_transcript_segments SET speaker_id = ?1 \+ WHERE id IN ( \+ SELECT mts.id FROM meeting_transcript_segments mts \+ JOIN audio_transcriptions at ON at.audio_chunk_id = ?2 \+ AND at.transcription = mts.transcript \+ AND ABS(julianday(at.timestamp) - julianday(mts.captured_at)) \+ <= 2.0 / 86400.0)",)- .bind(current_speaker_id)- .fetch_optional(&mut **tx.conn())+ .bind(target_speaker.id)+ .bind(audio_chunk_id)+ .execute(&mut **tx.conn()).await?;+ // 4. Move one embedding from old speaker to new speakerlet mut embeddings_moved = 0u64;- if let Some(emb_id) = embedding_id {- sqlx::query("UPDATE speaker_embeddings SET speaker_id = ? WHERE id = ?")- .bind(target_speaker.id)- .bind(emb_id)- .execute(&mut **tx.conn())- .await?;- embeddings_moved = 1;+ if let Some(current) = current_speaker_id {+ let embedding_id: Option<i64> = sqlx::query_scalar(+ "SELECT id FROM speaker_embeddings WHERE speaker_id = ? LIMIT 1",+ )+ .bind(current)+ .fetch_optional(&mut **tx.conn())+ .await?;++ if let Some(emb_id) = embedding_id {+ sqlx::query("UPDATE speaker_embeddings SET speaker_id = ? WHERE id = ?")+ .bind(target_speaker.id)+ .bind(emb_id)+ .execute(&mut **tx.conn())+ .await?;+ embeddings_moved = 1;+ }}tx.commit().await?;@@ -1128,17 +1152,17 @@ impl DatabaseManager {}// Phase 4: Clean up – if original speaker has no embeddings left, delete it- if current_speaker_id != target_speaker_id {+ if let Some(current) = current_speaker_id.filter(|&id| id != target_speaker_id) {let remaining: i64 =sqlx::query_scalar("SELECT COUNT(*) FROM speaker_embeddings WHERE speaker_id = ?")- .bind(current_speaker_id)+ .bind(current).fetch_one(&self.pool).await?;if remaining == 0 {let mut tx = self.begin_immediate_with_retry().await?;sqlx::query("DELETE FROM speakers WHERE id = ?")- .bind(current_speaker_id)+ .bind(current).execute(&mut **tx.conn()).await?;tx.commit().await?;diff --git a/crates/screenpipe-db/tests/speaker_reassignment_test.rs b/crates/screenpipe-db/tests/speaker_reassignment_test.rsindex 4fbbba866..ff6ffb2d6 100644--- a/crates/screenpipe-db/tests/speaker_reassignment_test.rs+++ b/crates/screenpipe-db/tests/speaker_reassignment_test.rs@@ -350,4 +350,205 @@ mod speaker_reassignment_tests {let alice_embedding_count = db.count_embeddings_for_speaker(alice_id).await.unwrap();assert_eq!(alice_embedding_count, 1); // Alice still has her original embedding}++ #[tokio::test]+ async fn test_reassign_speakerless_chunk() {+ let db = setup_test_db().await;++ // Mic rows and freshly-mirrored live rows carry speaker_id NULL until+ // backfill — reassigning them must work, not error out.+ let audio_chunk_id = db.insert_audio_chunk("mic_audio.mp4", None).await.unwrap();+ db.insert_audio_transcription(+ audio_chunk_id,+ "actually my colleague talking on my mic",+ 0,+ "",+ &AudioDevice {+ name: "test_mic".to_string(),+ device_type: DeviceType::Input,+ },+ None,+ Some(0.0),+ Some(5.0),+ None,+ )+ .await+ .unwrap();++ let (new_speaker_id, transcriptions_updated, embeddings_moved, old_assignments) = db+ .reassign_speaker(audio_chunk_id, "Dana", false)+ .await+ .unwrap();++ assert!(new_speaker_id > 0);+ assert_eq!(transcriptions_updated, 1);+ assert_eq!(embeddings_moved, 0); // no old speaker, nothing to move+ assert!(old_assignments.is_empty()); // NULL old speaker can't be undone++ let speaker = db.get_speaker_by_id(new_speaker_id).await.unwrap();+ assert_eq!(speaker.name, "Dana");+ }++ #[tokio::test]+ async fn test_reassign_syncs_meeting_transcript_segments() {+ let db = setup_test_db().await;++ // A live meeting segment and its mirrored audio_transcriptions row+ // share the same text and timestamp. Renaming via the chunk must also+ // relabel the segment, or the Meeting view keeps the old speaker.+ let meeting_id = db+ .insert_meeting("zoom", "test", Some("standup"), None)+ .await+ .unwrap();+ let captured_at = chrono::Utc::now();+ let segment_id = db+ .insert_meeting_transcript_segment(+ meeting_id,+ "deepgram",+ None,+ "item-1",+ "MacBook Pro Speakers",+ "output",+ Some("speaker 2"),+ "let's ship it tomorrow",+ captured_at,+ )+ .await+ .unwrap();+ assert!(segment_id > 0);++ let audio_chunk_id = db+ .insert_audio_chunk("macbook pro speakers (output)_123.mp4", None)+ .await+ .unwrap();+ db.insert_audio_transcription(+ audio_chunk_id,+ "let's ship it tomorrow",+ 0,+ "live",+ &AudioDevice {+ name: "MacBook Pro Speakers".to_string(),+ device_type: DeviceType::Output,+ },+ None,+ Some(0.0),+ Some(5.0),+ Some(captured_at),+ )+ .await+ .unwrap();++ let (new_speaker_id, _, _, _) = db+ .reassign_speaker(audio_chunk_id, "Priya", false)+ .await+ .unwrap();++ let segments = db+ .list_meeting_transcript_segments(meeting_id)+ .await+ .unwrap();+ let live: Vec<_> = segments.iter().filter(|s| s.source == "live").collect();+ assert_eq!(live.len(), 1);+ assert_eq!(live[0].speaker_id, Some(new_speaker_id));+ assert_eq!(live[0].speaker_name.as_deref(), Some("Priya"));+ }++ /// Helper: one live meeting segment + its mirrored audio_transcriptions row+ /// on the given chunk (same text, same timestamp — what+ /// `mirror_live_meeting_to_audio_transcriptions` produces).+ async fn seed_mirrored_segment(+ db: &DatabaseManager,+ meeting_id: i64,+ chunk_id: i64,+ label: &str,+ text: &str,+ captured_at: chrono::DateTime<chrono::Utc>,+ ) {+ db.insert_meeting_transcript_segment(+ meeting_id,+ "deepgram",+ None,+ &format!("item-{}-{}", chunk_id, text.replace(' ', "-")),+ "MacBook Pro Speakers",+ "output",+ Some(label),+ text,+ captured_at,+ )+ .await+ .unwrap();+ db.insert_audio_transcription(+ chunk_id,+ text,+ 0,+ "live",+ &AudioDevice {+ name: "MacBook Pro Speakers".to_string(),+ device_type: DeviceType::Output,+ },+ None,+ Some(0.0),+ Some(5.0),+ Some(captured_at),+ )+ .await+ .unwrap();+ }++ #[tokio::test]+ async fn test_live_segments_resolve_chunk_links_even_when_sharing_a_chunk() {+ let db = setup_test_db().await;++ // Several ~5s live segments get mirrored onto ONE ~30s audio chunk+ // (all offset_index 0). Every one of them must come back with the+ // chunk link, or the UI renders them as unrenameable static text.+ let meeting_id = db+ .insert_meeting("zoom", "test", Some("standup"), None)+ .await+ .unwrap();+ let chunk_id = db+ .insert_audio_chunk("macbook pro speakers (output)_456.mp4", None)+ .await+ .unwrap();+ let t0 = chrono::Utc::now();+ seed_mirrored_segment(&db, meeting_id, chunk_id, "speaker 1", "first utterance", t0).await;+ seed_mirrored_segment(+ &db,+ meeting_id,+ chunk_id,+ "speaker 2",+ "second utterance",+ t0 + chrono::Duration::seconds(5),+ )+ .await;+ seed_mirrored_segment(+ &db,+ meeting_id,+ chunk_id,+ "speaker 1",+ "third utterance",+ t0 + chrono::Duration::seconds(10),+ )+ .await;++ let segments = db+ .list_meeting_transcript_segments(meeting_id)+ .await+ .unwrap();+ let live: Vec<_> = segments.iter().filter(|s| s.source == "live").collect();+ assert_eq!(live.len(), 3);+ for seg in &live {+ assert_eq!(+ seg.audio_chunk_id,+ Some(chunk_id),+ "segment {:?} lost its chunk link",+ seg.transcript+ );+ assert_eq!(+ seg.audio_file_path.as_deref(),+ Some("macbook pro speakers (output)_456.mp4")+ );+ }+ }+}diff --git a/crates/screenpipe-db/tests/untranscribed_chunks_test.rs b/crates/screenpipe-db/tests/untranscribed_chunks_test.rsindex 20b84f105..434bf641d 100644--- a/crates/screenpipe-db/tests/untranscribed_chunks_test.rs+++ b/crates/screenpipe-db/tests/untranscribed_chunks_test.rs@@ -476,7 +476,11 @@ mod tests {assert_eq!(rows[1].source, "live");assert_eq!(rows[1].audio_chunk_id, None);- assert_eq!(rows[1].speaker_name.as_deref(), Some("Louis"));+ // Input (mic) rows expose a name only once a real global speaker is+ // assigned; the raw provider label is suppressed so the client renders+ // "me" — otherwise mic lines masked user assignments (unrenameable+ // "me" bug) or leaked diarization labels for the user's own voice.+ assert_eq!(rows[1].speaker_name, None);// The opposite-direction overlap row is KEPT (different participant), so it// sits between the live segment and the later background row.