YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(meetings): preserve live diarized transcript turns (#6625)
Ezra Elletted5fae4714cbd
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsxindex 8fc035c73..9ba38682d 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsx+++ b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsx@@ -22,11 +22,18 @@ vi.mock("@/components/rewind/media", () => ({}));import {+ applyLiveDelta,+ applyLiveFinal,+ coalesceFinalSpeakerRuns,+ filterBackgroundCoveredByLiveFinals,+ filterLiveCrossDeviceEchoes,isSpeakerContinuation,+ liveBlockToSpeakerBlock,SpeakerParagraph,TranscriptRows,type SpeakerBlock,} from "./transcript-panel";+import type { MeetingAudioChunk } from "@/lib/utils/meeting-context";const block: SpeakerBlock = {key: "chunk-1",@@ -141,6 +148,224 @@ describe("SpeakerParagraph render isolation", () => {).toBe(false);});+ it("keeps one moving realtime partial and retires it on finalization", () => {+ const firstDelta = {+ meeting_id: 1,+ provider: "deepgram",+ item_id: "partial-1",+ device_name: "System Audio",+ device_type: "output",+ delta: "hello",+ replace: true,+ captured_at: "2026-07-29T19:00:01.000Z",+ };+ const revised = applyLiveDelta([], firstDelta);+ const moved = applyLiveDelta(revised, {+ ...firstDelta,+ item_id: "partial-2",+ delta: "hello there",+ captured_at: "2026-07-29T19:00:05.000Z",+ });++ expect(moved).toHaveLength(1);+ expect(moved[0]).toMatchObject({+ itemId: "partial-2",+ text: "hello there",+ capturedAt: "2026-07-29T19:00:05.000Z",+ final: false,+ });++ const finalized = applyLiveFinal(moved, {+ meeting_id: 1,+ provider: "deepgram",+ item_id: "final-1",+ device_name: "System Audio",+ device_type: "output",+ speaker_name: "speaker 2",+ transcript: "Hello there.",+ captured_at: "2026-07-29T19:00:06.000Z",+ });+ expect(finalized).toHaveLength(1);+ expect(finalized[0]).toMatchObject({+ itemId: "final-1",+ speakerName: "speaker 2",+ final: true,+ });+ });++ it("merges adjacent finalized turns for one speaker but preserves real splits", () => {+ const sameSpeaker = {+ ...block,+ key: "speaker-2",+ startMs: block.endMs + 1_000,+ endMs: block.endMs + 1_000,+ text: "same speaker continuation",+ };+ const otherSpeaker = {+ ...sameSpeaker,+ key: "speaker-3",+ speakerId: 8,+ speakerKey: "speaker:8",+ speakerName: "other speaker",+ startMs: sameSpeaker.endMs + 1_000,+ endMs: sameSpeaker.endMs + 1_000,+ text: "interruption",+ };+ const firstRun = { ...block, speakerKey: "speaker:7" };++ expect(coalesceFinalSpeakerRuns([firstRun, sameSpeaker])).toMatchObject([+ {+ text: `${block.text} same speaker continuation`,+ segmentCount: 2,+ },+ ]);+ expect(+ coalesceFinalSpeakerRuns([firstRun, otherSpeaker, sameSpeaker]),+ ).toHaveLength(3);+ });++ it("preserves live provider labels and scopes them to the audio stream", () => {+ const remote = liveBlockToSpeakerBlock(+ {+ key: "output:deepgram:0",+ itemId: "deepgram:0",+ deviceName: "System Audio",+ deviceType: "output",+ speakerName: "speaker 2",+ provider: "deepgram",+ text: "remote participant",+ capturedAt: "2026-07-29T19:00:00.000Z",+ final: true,+ },+ 0,+ );+ const nearby = liveBlockToSpeakerBlock(+ {+ key: "input:deepgram:0",+ itemId: "deepgram:0",+ deviceName: "Built-in Mic",+ deviceType: "input",+ speakerName: "speaker 2",+ provider: "deepgram",+ text: "nearby participant",+ capturedAt: "2026-07-29T19:00:01.000Z",+ final: true,+ },+ 1,+ );++ expect(remote?.speakerName).toBe("speaker 2");+ expect(nearby?.speakerName).toBe("speaker 2");+ expect(remote?.speakerKey).not.toBe(nearby?.speakerKey);+ expect(+ isSpeakerContinuation(remote ?? undefined, nearby ?? undefined),+ ).toBe(false);+ });++ it("keeps live finals authoritative while retaining background gap fill", () => {+ const background = (overrides: Partial<MeetingAudioChunk>) => ({+ audioChunkId: 1,+ audioFilePath: "",+ speakerId: 99,+ speakerName: "wrong local match",+ deviceName: "Built-in Mic",+ deviceType: "input",+ isInput: true,+ transcription: "different batch words",+ timestamp: "2026-07-29T19:00:08.000Z",+ source: "background" as const,+ ...overrides,+ });+ const liveFinal = {+ key: "Built-in Mic:input:deepgram:0",+ itemId: "deepgram:0",+ deviceName: "Built-in Mic",+ deviceType: "input",+ speakerName: "speaker 1",+ provider: "deepgram",+ text: "authoritative live words",+ capturedAt: "2026-07-29T19:00:00.000Z",+ final: true,+ };++ const conflicting = background({});+ const gap = background({+ audioChunkId: 2,+ timestamp: "2026-07-29T19:00:30.000Z",+ transcription: "background gap fill",+ });+ const routedLive = background({+ audioChunkId: -3,+ source: "live",+ timestamp: "2026-07-29T19:00:08.000Z",+ transcription: "persisted live words",+ });++ expect(+ filterBackgroundCoveredByLiveFinals(+ [conflicting, gap, routedLive],+ [liveFinal],+ ),+ ).toEqual([gap, routedLive]);+ });++ it("suppresses short microphone echoes of clean system audio", () => {+ const output = {+ key: "output:1",+ itemId: "1",+ deviceName: "System Audio",+ deviceType: "output",+ speakerName: "speaker 1",+ provider: "deepgram",+ text: "What?",+ capturedAt: "2026-07-29T19:00:00.000Z",+ final: true,+ };+ const inputEcho = {+ ...output,+ key: "input:1",+ deviceName: "Built-in Mic",+ deviceType: "input",+ };+ const actualNearbySpeaker = {+ ...inputEcho,+ key: "input:2",+ text: "No.",+ };++ expect(+ filterLiveCrossDeviceEchoes([], [output, inputEcho, actualNearbySpeaker]),+ ).toEqual([output, actualNearbySpeaker]);+ });++ it("suppresses a short cached live suffix already present in its saved row", () => {+ const saved = {+ audioChunkId: -1,+ audioFilePath: "",+ speakerId: null,+ speakerName: "speaker 3",+ deviceName: "Built-in Mic",+ deviceType: "input",+ isInput: true,+ transcription: "Are you out of your mind?",+ timestamp: "2026-07-29T19:00:00.000Z",+ source: "live" as const,+ };+ const cachedSuffix = {+ key: "input:2",+ itemId: "2",+ deviceName: "Built-in Mic",+ deviceType: "input",+ speakerName: "speaker 3",+ provider: "deepgram",+ text: "your mind?",+ capturedAt: "2026-07-29T19:00:13.000Z",+ final: true,+ };++ expect(filterLiveCrossDeviceEchoes([saved], [cachedSuffix])).toEqual([]);+ });+it("uses monochrome theme tokens instead of blue or purple speaker colors", () => {const onSpeakerAssigned = vi.fn();const { getByTestId, rerender } = render(diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsxindex e63f004a8..2b6b9e7cf 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx+++ b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx@@ -85,7 +85,7 @@ function isNearBottom(el: HTMLDivElement): boolean {);}-interface LiveTranscriptDelta {+export interface LiveTranscriptDelta {meeting_id: number;provider: string;model?: string | null;@@ -97,13 +97,14 @@ interface LiveTranscriptDelta {captured_at: string;}-interface LiveTranscriptFinal {+export interface LiveTranscriptFinal {meeting_id: number;provider: string;model?: string | null;item_id: string;device_name: string;device_type: string;+ speaker_name?: string | null;transcript: string;captured_at: string;}@@ -125,11 +126,12 @@ interface LiveStreamingError {occurred_at: string;}-interface LiveTranscriptBlock {+export interface LiveTranscriptBlock {key: string;itemId: string;deviceName: string;deviceType: string;+ speakerName?: string | null;provider: string;model?: string | null;text: string;@@ -142,6 +144,8 @@ export interface SpeakerBlock {key: string;speakerId: number | null;speakerName: string;+ /** Provider labels are scoped to one audio stream, not global identities. */+ speakerKey?: string;startMs: number;endMs: number;text: string;@@ -156,6 +160,52 @@ export interface SpeakerBlock {const REFRESH_LIVE_MS = 30_000;const MAX_LIMIT = 5000;+const LIVE_TRANSCRIPT_CACHE_PREFIX = "screenpipe-meeting-live-finals:";+const LIVE_AUTHORITY_WINDOW_MS = 15_000;++function liveTranscriptCacheKey(meetingId: number): string {+ return `${LIVE_TRANSCRIPT_CACHE_PREFIX}${meetingId}`;+}++function loadCachedLiveFinals(meetingId: number): LiveTranscriptBlock[] {+ if (typeof window === "undefined") return [];+ try {+ const raw = window.sessionStorage.getItem(+ liveTranscriptCacheKey(meetingId),+ );+ if (!raw) return [];+ const parsed = JSON.parse(raw);+ return Array.isArray(parsed)+ ? parsed.filter(+ (block): block is LiveTranscriptBlock =>+ Boolean(block) &&+ block.final === true &&+ typeof block.key === "string" &&+ typeof block.text === "string" &&+ typeof block.capturedAt === "string",+ )+ : [];+ } catch {+ return [];+ }+}++function cacheLiveFinals(+ meetingId: number,+ blocks: LiveTranscriptBlock[],+): void {+ if (typeof window === "undefined") return;+ const finals = blocks.filter((block) => block.final);+ if (finals.length === 0) return;+ try {+ window.sessionStorage.setItem(+ liveTranscriptCacheKey(meetingId),+ JSON.stringify(finals),+ );+ } catch {+ // The DB remains the primary persistence path when web storage is blocked.+ }+}function liveKey(event: {item_id: string;@@ -165,6 +215,84 @@ function liveKey(event: {return `${event.device_name}:${event.device_type}:${event.item_id}`;}+function sameLiveStream(+ block: LiveTranscriptBlock,+ event: Pick<LiveTranscriptDelta, "provider" | "device_name" | "device_type">,+): boolean {+ return (+ block.provider === event.provider &&+ block.deviceName === event.device_name &&+ block.deviceType === event.device_type+ );+}++export function applyLiveDelta(+ blocks: LiveTranscriptBlock[],+ event: LiveTranscriptDelta,+): LiveTranscriptBlock[] {+ const delta = event.delta.trim();+ if (!delta) return blocks;+ const key = liveKey(event);+ const existing = blocks.find((block) => block.key === key);+ if (existing?.final) return blocks;+ if (existing) {+ return blocks.map((block) =>+ block.key === key+ ? {+ ...block,+ text: event.replace ? delta : `${block.text}${event.delta}`,+ capturedAt: event.captured_at,+ }+ : block,+ );+ }++ // Deepgram may revise the result start while an utterance is still open.+ // One stream has one active partial: retire its stale key before appending+ // the replacement so the realtime bubble cannot linger in the transcript.+ return [+ ...blocks.filter((block) => block.final || !sameLiveStream(block, event)),+ {+ key,+ itemId: event.item_id,+ deviceName: event.device_name,+ deviceType: event.device_type,+ provider: event.provider,+ model: event.model,+ text: delta,+ capturedAt: event.captured_at,+ final: false,+ },+ ];+}++export function applyLiveFinal(+ blocks: LiveTranscriptBlock[],+ event: LiveTranscriptFinal,+): LiveTranscriptBlock[] {+ const transcript = event.transcript.trim();+ if (!transcript) return blocks;+ const key = liveKey(event);+ return [+ ...blocks.filter(+ (block) =>+ block.key !== key && (block.final || !sameLiveStream(block, event)),+ ),+ {+ key,+ itemId: event.item_id,+ deviceName: event.device_name,+ deviceType: event.device_type,+ speakerName: event.speaker_name,+ provider: event.provider,+ model: event.model,+ text: transcript,+ capturedAt: event.captured_at,+ final: true,+ },+ ];+}+function normalizeForDedupe(text: string) {return text.toLowerCase().replace(/\s+/g, " ").trim();}@@ -174,6 +302,98 @@ function timestampMs(iso: string): number {return Number.isFinite(ms) ? ms : 0;}+/**+ * Keep provider-final live turns authoritative when the panel is reopened.+ * Background STT remains available only for gaps outside the live coverage+ * window; it must not replace a final with newly decoded words or speakers.+ */+export function filterBackgroundCoveredByLiveFinals(+ chunks: MeetingAudioChunk[],+ liveBlocks: LiveTranscriptBlock[],+): MeetingAudioChunk[] {+ const finals = liveBlocks+ .filter((block) => block.final)+ .map((block) => ({+ timestamp: timestampMs(block.capturedAt),+ deviceName: block.deviceName.trim().toLowerCase(),+ deviceType: block.deviceType.trim().toLowerCase(),+ }))+ .filter((block) => block.timestamp > 0);+ if (finals.length === 0) return chunks;++ return chunks.filter((chunk) => {+ if (chunk.source === "live") return true;+ const timestamp = timestampMs(chunk.timestamp);+ const deviceName = chunk.deviceName?.trim().toLowerCase() ?? "";+ const deviceType = chunk.deviceType.trim().toLowerCase();+ return !finals.some(+ (final) =>+ final.deviceType === deviceType &&+ (!final.deviceName || !deviceName || final.deviceName === deviceName) &&+ Math.abs(final.timestamp - timestamp) <= LIVE_AUTHORITY_WINDOW_MS,+ );+ });+}++/** Prefer clean system audio when the microphone hears the same nearby words. */+export function filterLiveCrossDeviceEchoes(+ chunks: MeetingAudioChunk[],+ liveBlocks: LiveTranscriptBlock[],+): LiveTranscriptBlock[] {+ const echoWindowMs = 6_000;+ const durableWindowMs = 15_000;+ const durableBlocks = chunks+ .map((chunk) => ({+ timestamp: timestampMs(chunk.timestamp),+ deviceName: chunk.deviceName?.trim().toLowerCase() ?? "",+ deviceType: chunk.deviceType.trim().toLowerCase(),+ text: normalizeForDedupe(chunk.transcription ?? ""),+ }))+ .filter((block) => block.timestamp > 0 && block.text.length > 0);+ const outputBlocks = liveBlocks+ .filter((block) => block.deviceType.toLowerCase() === "output")+ .map((block) => ({+ timestamp: timestampMs(block.capturedAt),+ words: normalizeForDedupe(block.text).split(" ").filter(Boolean),+ }));++ return liveBlocks.filter((block) => {+ const normalized = normalizeForDedupe(block.text);+ const timestamp = timestampMs(block.capturedAt);+ const deviceName = block.deviceName.trim().toLowerCase();+ const deviceType = block.deviceType.trim().toLowerCase();+ const alreadyDurable = durableBlocks.some(+ (durable) =>+ durable.deviceType === deviceType &&+ (!durable.deviceName ||+ !deviceName ||+ durable.deviceName === deviceName) &&+ Math.abs(durable.timestamp - timestamp) <= durableWindowMs &&+ (durable.text.includes(normalized) ||+ normalized.includes(durable.text)),+ );+ if (normalized && alreadyDurable) return false;++ if (block.deviceType.toLowerCase() === "input") {+ const nearbyOutputWords = new Set(+ outputBlocks+ .filter(+ (output) => Math.abs(output.timestamp - timestamp) <= echoWindowMs,+ )+ .flatMap((output) => output.words),+ );+ const words = normalized.split(" ").filter(Boolean);+ const covered =+ words.length > 0+ ? words.filter((word) => nearbyOutputWords.has(word)).length /+ words.length+ : 0;+ if (covered >= 0.6) return false;+ }+ return true;+ });+}+function sortChunks(chunks: MeetingAudioChunk[]): MeetingAudioChunk[] {return [...chunks].sort((a, b) => {const byTime = timestampMs(a.timestamp) - timestampMs(b.timestamp);@@ -203,9 +423,12 @@ function groupBySpeaker(chunks: MeetingAudioChunk[]): SpeakerBlock[] {if (ts <= 0) continue;const speakerName = c.speakerName || (c.isInput ? "me" : "speaker");const speakerId = c.speakerId;+ const speakerKey =+ speakerId != null+ ? `speaker:${speakerId}`+ : `stream:${c.deviceName || c.deviceType}:${speakerName}`;const last = out[out.length - 1];- const sameSpeaker =- last && last.speakerId === speakerId && last.speakerName === speakerName;+ const sameSpeaker = last?.speakerKey === speakerKey;// Glue if same speaker AND within 30s of last segment — keeps long pauses// as paragraph breaks even when the same person is still talking.if (sameSpeaker && ts - last.endMs < 30_000) {@@ -217,6 +440,7 @@ function groupBySpeaker(chunks: MeetingAudioChunk[]): SpeakerBlock[] {key: `${c.audioChunkId}-${ts}-${out.length}`,speakerId,speakerName,+ speakerKey,startMs: ts,endMs: ts,text,@@ -231,7 +455,7 @@ function groupBySpeaker(chunks: MeetingAudioChunk[]): SpeakerBlock[] {return out;}-function liveBlockToSpeakerBlock(+export function liveBlockToSpeakerBlock(block: LiveTranscriptBlock,index: number,): SpeakerBlock | null {@@ -241,7 +465,14 @@ function liveBlockToSpeakerBlock(return {key: `live-${block.key}-${index}`,speakerId: null,- speakerName: block.deviceType.toLowerCase() === "input" ? "me" : "speaker",+ speakerName:+ block.speakerName?.trim() ||+ (!block.final+ ? "transcribing"+ : block.deviceType.toLowerCase() === "input"+ ? "me"+ : "speaker"),+ speakerKey: `stream:${block.deviceName}:${block.deviceType}:${block.speakerName?.trim() || "unknown"}`,startMs,endMs: startMs,text,@@ -308,7 +539,9 @@ export function TranscriptPanel({const [query, setQuery] = useState("");const [searchOpen, setSearchOpen] = useState(false);const searchInputRef = useRef<HTMLInputElement | null>(null);- const [liveBlocks, setLiveBlocks] = useState<LiveTranscriptBlock[]>([]);+ const [liveBlocks, setLiveBlocks] = useState<LiveTranscriptBlock[]>(() =>+ loadCachedLiveFinals(meeting.id),+ );const [liveStatus, setLiveStatus] = useState<LiveStreamingStatus | null>(null,);@@ -434,7 +667,7 @@ export function TranscriptPanel({}, [meeting.id, range.start, range.end, refreshKey]);useEffect(() => {- setLiveBlocks([]);+ setLiveBlocks(loadCachedLiveFinals(meeting.id));setLiveStatus(null);setLiveError(null);}, [meeting.id]);@@ -451,34 +684,7 @@ export function TranscriptPanel({const delta = event.payload.delta ?? "";if (!delta.trim()) return;setLiveError(null);- const key = liveKey(event.payload);- setLiveBlocks((prev) => {- const existing = prev.find((b) => b.key === key);- if (existing) {- return prev.map((b) =>- b.key === key- ? {- ...b,- text: event.payload.replace ? delta : `${b.text}${delta}`,- }- : b,- );- }- return [- ...prev,- {- key,- itemId: event.payload.item_id,- deviceName: event.payload.device_name,- deviceType: event.payload.device_type,- provider: event.payload.provider,- model: event.payload.model,- text: delta,- capturedAt: event.payload.captured_at,- final: false,- },- ];- });+ setLiveBlocks((prev) => applyLiveDelta(prev, event.payload));},);@@ -487,37 +693,12 @@ export function TranscriptPanel({(event) => {if (cancelled || Number(event.payload.meeting_id) !== meeting.id)return;- const transcript = (event.payload.transcript ?? "").trim();- if (!transcript) return;+ if (!(event.payload.transcript ?? "").trim()) return;setLiveError(null);- const key = liveKey(event.payload);setLiveBlocks((prev) => {- const existing = prev.find((b) => b.key === key);- if (existing) {- return prev.map((b) =>- b.key === key- ? {- ...b,- text: transcript,- final: true,- }- : b,- );- }- return [- ...prev,- {- key,- itemId: event.payload.item_id,- deviceName: event.payload.device_name,- deviceType: event.payload.device_type,- provider: event.payload.provider,- model: event.payload.model,- text: transcript,- capturedAt: event.payload.captured_at,- final: true,- },- ];+ const next = applyLiveFinal(prev, event.payload);+ cacheLiveFinals(meeting.id, next);+ return next;});},);@@ -586,51 +767,18 @@ export function TranscriptPanel({};}, [isOpen, meeting.id, range.start, range.end, isLive, refreshKey]);- const blocks = useMemo(() => groupBySpeaker(chunks), [chunks]);- const visibleLiveBlocks = useMemo(() => {- const durableText = normalizeForDedupe(- chunks.map((c) => c.transcription ?? "").join(" "),- );- // Cross-device echo suppression. Without headphones the mic ("input"/"me")- // picks up the speaker output, so the remote's words arrive on BOTH the- // input stream and the clean system-audio ("output"/"speaker") stream.- // macOS VoiceProcessingIO AEC does not remove this (it has no downlink- // reference), and the engine's cross-device dedup only runs on the deferred- // durable path — so during a live meeting both copies reach the UI. The- // output capture is the clean source, so drop an input block when most of- // its words are covered by a nearby output block.- const ECHO_WINDOW_MS = 6000;- const outputBlocks = liveBlocks- .filter((b) => b.deviceType.toLowerCase() === "output")- .map((b) => ({- ts: timestampMs(b.capturedAt),- norm: normalizeForDedupe(b.text),- }));- const isInputEchoOfOutput = (- block: LiveTranscriptBlock,- normalized: string,- ) => {- if (block.deviceType.toLowerCase() !== "input") return false;- const ts = timestampMs(block.capturedAt);- const ref = new Set(- outputBlocks- .filter((o) => Math.abs(o.ts - ts) <= ECHO_WINDOW_MS)- .flatMap((o) => o.norm.split(" "))- .filter(Boolean),- );- if (ref.size === 0) return false;- const words = normalized.split(" ").filter(Boolean);- if (words.length === 0) return false;- const covered = words.filter((w) => ref.has(w)).length / words.length;- return covered >= 0.6;- };- return liveBlocks.filter((block) => {- const normalized = normalizeForDedupe(block.text);- if (normalized.length < 24) return true;- if (durableText.includes(normalized.slice(0, 80))) return false;- return !isInputEchoOfOutput(block, normalized);- });- }, [chunks, liveBlocks]);+ const authoritativeChunks = useMemo(+ () => filterBackgroundCoveredByLiveFinals(chunks, liveBlocks),+ [chunks, liveBlocks],+ );+ const blocks = useMemo(+ () => groupBySpeaker(authoritativeChunks),+ [authoritativeChunks],+ );+ const visibleLiveBlocks = useMemo(+ () => filterLiveCrossDeviceEchoes(authoritativeChunks, liveBlocks),+ [authoritativeChunks, liveBlocks],+ );const visibleLiveSpeakerBlocks = useMemo(() =>visibleLiveBlocks@@ -639,7 +787,10 @@ export function TranscriptPanel({[visibleLiveBlocks],);const displayBlocks = useMemo(- () => [...blocks, ...visibleLiveSpeakerBlocks].sort(compareBlocks),+ () =>+ coalesceFinalSpeakerRuns(+ [...blocks, ...visibleLiveSpeakerBlocks].sort(compareBlocks),+ ),[blocks, visibleLiveSpeakerBlocks],);const latestBlockSignal = useMemo(() => {@@ -962,7 +1113,11 @@ export function TranscriptPanel({"h-7 w-7 p-0",searchOpen && "bg-accent text-accent-foreground",)}- title={searchOpen ? "hide search" : `search transcript (${isMac ? "⌘F" : "Ctrl+F"})`}+ title={+ searchOpen+ ? "hide search"+ : `search transcript (${isMac ? "⌘F" : "Ctrl+F"})`+ }aria-label={searchOpen ? "hide transcript search" : "search transcript"}@@ -1260,14 +1415,39 @@ export const SpeakerParagraph = React.memo(function SpeakerParagraph({const SPEAKER_RUN_MAX_GAP_MS = 30_000;+export function coalesceFinalSpeakerRuns(+ blocks: SpeakerBlock[],+): SpeakerBlock[] {+ const merged: SpeakerBlock[] = [];+ for (const block of blocks) {+ const previous = merged[merged.length - 1];+ if (+ previous?.final &&+ block.final &&+ isSpeakerContinuation(previous, block)+ ) {+ previous.text = `${previous.text} ${block.text}`;+ previous.endMs = Math.max(previous.endMs, block.endMs);+ previous.segmentCount += block.segmentCount;+ continue;+ }+ merged.push({ ...block });+ }+ return merged;+}+export function isSpeakerContinuation(previous: SpeakerBlock | undefined,current: SpeakerBlock | undefined,): boolean {if (!previous || !current) return false;+ const sameSpeaker =+ previous.speakerKey && current.speakerKey+ ? previous.speakerKey === current.speakerKey+ : previous.speakerId === current.speakerId &&+ previous.speakerName === current.speakerName;return (- previous.speakerId === current.speakerId &&- previous.speakerName === current.speakerName &&+ sameSpeaker &¤t.startMs >= previous.endMs &¤t.startMs - previous.endMs <= SPEAKER_RUN_MAX_GAP_MS);diff --git a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts b/apps/screenpipe-app-tauri/lib/utils/meeting-context.tsindex 348697ffc..ac3b7db2f 100644--- a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts+++ b/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts@@ -37,6 +37,7 @@ export interface MeetingAudioChunk {audioFilePath: string;speakerId: number | null;speakerName: string;+ deviceName?: string;deviceType: string;isInput: boolean;transcription: string;@@ -853,6 +854,7 @@ interface SearchAudioItem {transcription?: string;timestamp?: string;file_path?: string;+ device?: string;device_type?: string;speaker?: { id?: number; name?: string } | null;};@@ -904,6 +906,7 @@ export async function fetchMeetingAudio(// Mic rows show "me" only until someone is actually assigned —// hardcoding "me" made input lines look impossible to reassign.speakerName: c.speaker?.name?.trim() || (isInput ? "me" : ""),+ deviceName: c.device,deviceType,isInput,transcription: c.transcription,@@ -982,6 +985,7 @@ async function fetchRoutedMeetingTranscript(// masked by a hardcoded "me".speakerName:segment.speakerName?.trim() || (isInput ? "me" : "speaker"),+ deviceName: segment.deviceName,deviceType,isInput,transcription: segment.transcript,diff --git a/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rs b/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rsindex 8f2df47fb..3bb9954bc 100644--- a/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rs+++ b/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rs@@ -366,19 +366,37 @@ fn handle_results_event(let captured_at = latest_audio_time(latest_audio_ms);if is_final {- let speaker_name = resolve_speaker_name(value, device_type, config);- let event = MeetingTranscriptFinal {- meeting_id,- provider: config.provider.as_str().to_string(),- model: config.model.clone(),- item_id,- device_name: device_name.to_string(),- device_type: device_type.to_string(),- speaker_name,- transcript: transcript.to_string(),- captured_at,- };- let _ = screenpipe_events::send_event("meeting_transcript_final", event);+ let turns = speaker_turns(value);+ if turns.is_empty() {+ emit_final(+ meeting_id,+ config,+ device_name,+ device_type,+ item_id,+ None,+ transcript.to_string(),+ captured_at,+ );+ } else {+ for (index, turn) in turns.into_iter().enumerate() {+ let turn_item_id = if index == 0 {+ item_id.clone()+ } else {+ format!("{item_id}:turn:{index}")+ };+ emit_final(+ meeting_id,+ config,+ device_name,+ device_type,+ turn_item_id,+ Some(format!("speaker {}", turn.speaker + 1)),+ turn.transcript,+ captured_at + chrono::Duration::milliseconds(index as i64),+ );+ }+ }} else {let event = MeetingTranscriptDelta {meeting_id,@@ -395,34 +413,74 @@ fn handle_results_event(}}-/// Pick a speaker label from a Deepgram `Results` payload with `diarize=true`.-/// Counts speaker indices across the words array and returns the dominant one.-/// `device_type == "input"` + `speaker 0` is treated as the local user when-/// `local_speaker_name` is set, since deepgram numbers speakers in order of-/// first utterance and the local user typically speaks first into their own-/// mic. All other indices fall back to a generic `speaker N` (1-indexed).-fn resolve_speaker_name(- value: &Value,- device_type: &str,+fn emit_final(+ meeting_id: i64,config: &MeetingStreamingConfig,-) -> Option<String> {- let words = value+ device_name: &str,+ device_type: &str,+ item_id: String,+ speaker_name: Option<String>,+ transcript: String,+ captured_at: DateTime<Utc>,+) {+ let event = MeetingTranscriptFinal {+ meeting_id,+ provider: config.provider.as_str().to_string(),+ model: config.model.clone(),+ item_id,+ device_name: device_name.to_string(),+ device_type: device_type.to_string(),+ speaker_name,+ transcript,+ captured_at,+ };+ let _ = screenpipe_events::send_event("meeting_transcript_final", event);+}++#[derive(Debug, PartialEq, Eq)]+struct DeepgramSpeakerTurn {+ speaker: i64,+ transcript: String,+}++/// Preserve Deepgram's consecutive per-word speaker runs instead of assigning+/// a mixed-speaker result to whichever speaker happened to say more words.+///+/// A partial diarization payload falls back to the top-level transcript rather+/// than dropping unlabelled words or guessing their speaker.+fn speaker_turns(value: &Value) -> Vec<DeepgramSpeakerTurn> {+ let Some(words) = value.pointer("/channel/alternatives/0/words")- .and_then(Value::as_array)?;- let mut counts: std::collections::HashMap<i64, usize> = std::collections::HashMap::new();+ .and_then(Value::as_array)+ else {+ return Vec::new();+ };+ let mut turns: Vec<DeepgramSpeakerTurn> = Vec::new();for word in words {- if let Some(speaker) = word.get("speaker").and_then(Value::as_i64) {- *counts.entry(speaker).or_insert(0) += 1;- }- }- let (dominant, _) = counts.into_iter().max_by_key(|&(_, n)| n)?;+ let Some(text) = word+ .get("punctuated_word")+ .and_then(Value::as_str)+ .or_else(|| word.get("word").and_then(Value::as_str))+ .map(str::trim)+ .filter(|text| !text.is_empty())+ else {+ continue;+ };+ let Some(speaker) = word.get("speaker").and_then(Value::as_i64) else {+ return Vec::new();+ };- if dominant == 0 && device_type == "input" {- if let Some(name) = config.local_speaker_name.clone() {- return Some(name);+ if let Some(turn) = turns.last_mut().filter(|turn| turn.speaker == speaker) {+ turn.transcript.push(' ');+ turn.transcript.push_str(text);+ } else {+ turns.push(DeepgramSpeakerTurn {+ speaker,+ transcript: text.to_string(),+ });}}- Some(format!("speaker {}", dominant + 1))+ turns}fn latest_audio_time(latest_audio_ms: &AtomicU64) -> DateTime<Utc> {@@ -476,6 +534,48 @@ fn device_type_label(device_type: &DeviceType) -> &'static str {mod tests {use super::*;+ #[test]+ fn preserves_consecutive_deepgram_speaker_turns() {+ let payload = json!({+ "channel": { "alternatives": [{ "words": [+ { "word": "hello", "punctuated_word": "Hello,", "speaker": 0 },+ { "word": "there", "punctuated_word": "there.", "speaker": 0 },+ { "word": "hi", "punctuated_word": "Hi!", "speaker": 1 },+ { "word": "yes", "punctuated_word": "Yes.", "speaker": 0 }+ ]}]}+ });++ assert_eq!(+ speaker_turns(&payload),+ vec![+ DeepgramSpeakerTurn {+ speaker: 0,+ transcript: "Hello, there.".to_string(),+ },+ DeepgramSpeakerTurn {+ speaker: 1,+ transcript: "Hi!".to_string(),+ },+ DeepgramSpeakerTurn {+ speaker: 0,+ transcript: "Yes.".to_string(),+ },+ ]+ );+ }++ #[test]+ fn incomplete_diarization_falls_back_without_dropping_words() {+ let payload = json!({+ "channel": { "alternatives": [{ "words": [+ { "word": "labelled", "speaker": 0 },+ { "word": "unknown" }+ ]}]}+ });++ assert!(speaker_turns(&payload).is_empty());+ }+fn live_query(language: Option<&str>) -> String {let config = MeetingStreamingConfig {enabled: true,diff --git a/crates/screenpipe-db/src/db/meetings.rs b/crates/screenpipe-db/src/db/meetings.rsindex b89dc169d..1fbc0b1e0 100644--- a/crates/screenpipe-db/src/db/meetings.rs+++ b/crates/screenpipe-db/src/db/meetings.rs@@ -1184,15 +1184,11 @@ impl DatabaseManager {NULL AS audio_chunk_id,NULL AS audio_file_path,mts.speaker_id AS speaker_id,- -- Prefer the resolved global speaker's name; output rows fall- -- back to the free-text Deepgram label until backfilled / if- -- the speaker is unnamed (NULLIF treats '' as "no name yet").- -- Input (mic) rows get NULL instead of the raw label so the- -- client can render "me" until the user assigns someone.- CASE- WHEN mts.device_type = 'input' THEN NULLIF(s.name, '')- ELSE COALESCE(NULLIF(s.name, ''), mts.speaker_name)- END AS speaker_name,+ -- Prefer the resolved global speaker's name, then preserve+ -- Deepgram's stream-local label until voice backfill gives it+ -- a durable identity. A mic is a capture source, not proof+ -- that every voice on it belongs to the local user.+ COALESCE(NULLIF(s.name, ''), mts.speaker_name) AS speaker_name,mts.transcript,mts.captured_at,mts.created_atdiff --git a/crates/screenpipe-db/tests/timeline_live_meeting_test.rs b/crates/screenpipe-db/tests/timeline_live_meeting_test.rsindex 625912a2a..9dd920708 100644--- a/crates/screenpipe-db/tests/timeline_live_meeting_test.rs+++ b/crates/screenpipe-db/tests/timeline_live_meeting_test.rs@@ -499,9 +499,10 @@ mod timeline_live_meeting_tests {}/// Until a segment is resolved, the Meeting view falls back to Deepgram's- /// free-text `speaker_name`, and the backfill maps nothing.+ /// free-text `speaker_name` even for mic input. A capture source is not a+ /// speaker identity, and the backfill may have nothing to map yet.#[tokio::test]- async fn test_meeting_segment_falls_back_to_freetext_speaker() {+ async fn test_input_meeting_segment_falls_back_to_freetext_speaker() {let db = setup_test_db().await;let base = Utc::now();@@ -514,8 +515,8 @@ mod timeline_live_meeting_tests {"screenpipe-cloud",None,"deepgram:0:0",- "System Audio",- "output",+ "Built-in Mic",+ "input",Some("speaker 2"),"unresolved line",base,