Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(meetings): preserve live diarized transcript turns (#6625)

Ezra Elletted5fae4714cbd
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsx
index 8fc035c73..9ba38682d 100644
--- a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsx
+++ b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel-performance.test.tsx
@@ -22,11 +22,18 @@ vi.mock("@/components/rewind/media", () => ({
}));
import {
+ applyLiveDelta,
+ applyLiveFinal,
+ coalesceFinalSpeakerRuns,
+ filterBackgroundCoveredByLiveFinals,
+ filterLiveCrossDeviceEchoes,
isSpeakerContinuation,
+ liveBlockToSpeakerBlock,
SpeakerParagraph,
TranscriptRows,
type SpeakerBlock,
} from "./transcript-panel";
+import type { MeetingAudioChunk } from "@/lib/utils/meeting-context";
const block: SpeakerBlock = {
key: "chunk-1",
@@ -141,6 +148,224 @@ describe("SpeakerParagraph render isolation", () => {
).toBe(false);
});
+ it("keeps one moving realtime partial and retires it on finalization", () => {
+ const firstDelta = {
+ meeting_id: 1,
+ provider: "deepgram",
+ item_id: "partial-1",
+ device_name: "System Audio",
+ device_type: "output",
+ delta: "hello",
+ replace: true,
+ captured_at: "2026-07-29T19:00:01.000Z",
+ };
+ const revised = applyLiveDelta([], firstDelta);
+ const moved = applyLiveDelta(revised, {
+ ...firstDelta,
+ item_id: "partial-2",
+ delta: "hello there",
+ captured_at: "2026-07-29T19:00:05.000Z",
+ });
+
+ expect(moved).toHaveLength(1);
+ expect(moved[0]).toMatchObject({
+ itemId: "partial-2",
+ text: "hello there",
+ capturedAt: "2026-07-29T19:00:05.000Z",
+ final: false,
+ });
+
+ const finalized = applyLiveFinal(moved, {
+ meeting_id: 1,
+ provider: "deepgram",
+ item_id: "final-1",
+ device_name: "System Audio",
+ device_type: "output",
+ speaker_name: "speaker 2",
+ transcript: "Hello there.",
+ captured_at: "2026-07-29T19:00:06.000Z",
+ });
+ expect(finalized).toHaveLength(1);
+ expect(finalized[0]).toMatchObject({
+ itemId: "final-1",
+ speakerName: "speaker 2",
+ final: true,
+ });
+ });
+
+ it("merges adjacent finalized turns for one speaker but preserves real splits", () => {
+ const sameSpeaker = {
+ ...block,
+ key: "speaker-2",
+ startMs: block.endMs + 1_000,
+ endMs: block.endMs + 1_000,
+ text: "same speaker continuation",
+ };
+ const otherSpeaker = {
+ ...sameSpeaker,
+ key: "speaker-3",
+ speakerId: 8,
+ speakerKey: "speaker:8",
+ speakerName: "other speaker",
+ startMs: sameSpeaker.endMs + 1_000,
+ endMs: sameSpeaker.endMs + 1_000,
+ text: "interruption",
+ };
+ const firstRun = { ...block, speakerKey: "speaker:7" };
+
+ expect(coalesceFinalSpeakerRuns([firstRun, sameSpeaker])).toMatchObject([
+ {
+ text: `${block.text} same speaker continuation`,
+ segmentCount: 2,
+ },
+ ]);
+ expect(
+ coalesceFinalSpeakerRuns([firstRun, otherSpeaker, sameSpeaker]),
+ ).toHaveLength(3);
+ });
+
+ it("preserves live provider labels and scopes them to the audio stream", () => {
+ const remote = liveBlockToSpeakerBlock(
+ {
+ key: "output:deepgram:0",
+ itemId: "deepgram:0",
+ deviceName: "System Audio",
+ deviceType: "output",
+ speakerName: "speaker 2",
+ provider: "deepgram",
+ text: "remote participant",
+ capturedAt: "2026-07-29T19:00:00.000Z",
+ final: true,
+ },
+ 0,
+ );
+ const nearby = liveBlockToSpeakerBlock(
+ {
+ key: "input:deepgram:0",
+ itemId: "deepgram:0",
+ deviceName: "Built-in Mic",
+ deviceType: "input",
+ speakerName: "speaker 2",
+ provider: "deepgram",
+ text: "nearby participant",
+ capturedAt: "2026-07-29T19:00:01.000Z",
+ final: true,
+ },
+ 1,
+ );
+
+ expect(remote?.speakerName).toBe("speaker 2");
+ expect(nearby?.speakerName).toBe("speaker 2");
+ expect(remote?.speakerKey).not.toBe(nearby?.speakerKey);
+ expect(
+ isSpeakerContinuation(remote ?? undefined, nearby ?? undefined),
+ ).toBe(false);
+ });
+
+ it("keeps live finals authoritative while retaining background gap fill", () => {
+ const background = (overrides: Partial<MeetingAudioChunk>) => ({
+ audioChunkId: 1,
+ audioFilePath: "",
+ speakerId: 99,
+ speakerName: "wrong local match",
+ deviceName: "Built-in Mic",
+ deviceType: "input",
+ isInput: true,
+ transcription: "different batch words",
+ timestamp: "2026-07-29T19:00:08.000Z",
+ source: "background" as const,
+ ...overrides,
+ });
+ const liveFinal = {
+ key: "Built-in Mic:input:deepgram:0",
+ itemId: "deepgram:0",
+ deviceName: "Built-in Mic",
+ deviceType: "input",
+ speakerName: "speaker 1",
+ provider: "deepgram",
+ text: "authoritative live words",
+ capturedAt: "2026-07-29T19:00:00.000Z",
+ final: true,
+ };
+
+ const conflicting = background({});
+ const gap = background({
+ audioChunkId: 2,
+ timestamp: "2026-07-29T19:00:30.000Z",
+ transcription: "background gap fill",
+ });
+ const routedLive = background({
+ audioChunkId: -3,
+ source: "live",
+ timestamp: "2026-07-29T19:00:08.000Z",
+ transcription: "persisted live words",
+ });
+
+ expect(
+ filterBackgroundCoveredByLiveFinals(
+ [conflicting, gap, routedLive],
+ [liveFinal],
+ ),
+ ).toEqual([gap, routedLive]);
+ });
+
+ it("suppresses short microphone echoes of clean system audio", () => {
+ const output = {
+ key: "output:1",
+ itemId: "1",
+ deviceName: "System Audio",
+ deviceType: "output",
+ speakerName: "speaker 1",
+ provider: "deepgram",
+ text: "What?",
+ capturedAt: "2026-07-29T19:00:00.000Z",
+ final: true,
+ };
+ const inputEcho = {
+ ...output,
+ key: "input:1",
+ deviceName: "Built-in Mic",
+ deviceType: "input",
+ };
+ const actualNearbySpeaker = {
+ ...inputEcho,
+ key: "input:2",
+ text: "No.",
+ };
+
+ expect(
+ filterLiveCrossDeviceEchoes([], [output, inputEcho, actualNearbySpeaker]),
+ ).toEqual([output, actualNearbySpeaker]);
+ });
+
+ it("suppresses a short cached live suffix already present in its saved row", () => {
+ const saved = {
+ audioChunkId: -1,
+ audioFilePath: "",
+ speakerId: null,
+ speakerName: "speaker 3",
+ deviceName: "Built-in Mic",
+ deviceType: "input",
+ isInput: true,
+ transcription: "Are you out of your mind?",
+ timestamp: "2026-07-29T19:00:00.000Z",
+ source: "live" as const,
+ };
+ const cachedSuffix = {
+ key: "input:2",
+ itemId: "2",
+ deviceName: "Built-in Mic",
+ deviceType: "input",
+ speakerName: "speaker 3",
+ provider: "deepgram",
+ text: "your mind?",
+ capturedAt: "2026-07-29T19:00:13.000Z",
+ final: true,
+ };
+
+ expect(filterLiveCrossDeviceEchoes([saved], [cachedSuffix])).toEqual([]);
+ });
+
it("uses monochrome theme tokens instead of blue or purple speaker colors", () => {
const onSpeakerAssigned = vi.fn();
const { getByTestId, rerender } = render(
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx
index e63f004a8..2b6b9e7cf 100644
--- a/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx
+++ b/apps/screenpipe-app-tauri/components/meeting-notes/transcript-panel.tsx
@@ -85,7 +85,7 @@ function isNearBottom(el: HTMLDivElement): boolean {
);
}
-interface LiveTranscriptDelta {
+export interface LiveTranscriptDelta {
meeting_id: number;
provider: string;
model?: string | null;
@@ -97,13 +97,14 @@ interface LiveTranscriptDelta {
captured_at: string;
}
-interface LiveTranscriptFinal {
+export interface LiveTranscriptFinal {
meeting_id: number;
provider: string;
model?: string | null;
item_id: string;
device_name: string;
device_type: string;
+ speaker_name?: string | null;
transcript: string;
captured_at: string;
}
@@ -125,11 +126,12 @@ interface LiveStreamingError {
occurred_at: string;
}
-interface LiveTranscriptBlock {
+export interface LiveTranscriptBlock {
key: string;
itemId: string;
deviceName: string;
deviceType: string;
+ speakerName?: string | null;
provider: string;
model?: string | null;
text: string;
@@ -142,6 +144,8 @@ export interface SpeakerBlock {
key: string;
speakerId: number | null;
speakerName: string;
+ /** Provider labels are scoped to one audio stream, not global identities. */
+ speakerKey?: string;
startMs: number;
endMs: number;
text: string;
@@ -156,6 +160,52 @@ export interface SpeakerBlock {
const REFRESH_LIVE_MS = 30_000;
const MAX_LIMIT = 5000;
+const LIVE_TRANSCRIPT_CACHE_PREFIX = "screenpipe-meeting-live-finals:";
+const LIVE_AUTHORITY_WINDOW_MS = 15_000;
+
+function liveTranscriptCacheKey(meetingId: number): string {
+ return `${LIVE_TRANSCRIPT_CACHE_PREFIX}${meetingId}`;
+}
+
+function loadCachedLiveFinals(meetingId: number): LiveTranscriptBlock[] {
+ if (typeof window === "undefined") return [];
+ try {
+ const raw = window.sessionStorage.getItem(
+ liveTranscriptCacheKey(meetingId),
+ );
+ if (!raw) return [];
+ const parsed = JSON.parse(raw);
+ return Array.isArray(parsed)
+ ? parsed.filter(
+ (block): block is LiveTranscriptBlock =>
+ Boolean(block) &&
+ block.final === true &&
+ typeof block.key === "string" &&
+ typeof block.text === "string" &&
+ typeof block.capturedAt === "string",
+ )
+ : [];
+ } catch {
+ return [];
+ }
+}
+
+function cacheLiveFinals(
+ meetingId: number,
+ blocks: LiveTranscriptBlock[],
+): void {
+ if (typeof window === "undefined") return;
+ const finals = blocks.filter((block) => block.final);
+ if (finals.length === 0) return;
+ try {
+ window.sessionStorage.setItem(
+ liveTranscriptCacheKey(meetingId),
+ JSON.stringify(finals),
+ );
+ } catch {
+ // The DB remains the primary persistence path when web storage is blocked.
+ }
+}
function liveKey(event: {
item_id: string;
@@ -165,6 +215,84 @@ function liveKey(event: {
return `${event.device_name}:${event.device_type}:${event.item_id}`;
}
+function sameLiveStream(
+ block: LiveTranscriptBlock,
+ event: Pick<LiveTranscriptDelta, "provider" | "device_name" | "device_type">,
+): boolean {
+ return (
+ block.provider === event.provider &&
+ block.deviceName === event.device_name &&
+ block.deviceType === event.device_type
+ );
+}
+
+export function applyLiveDelta(
+ blocks: LiveTranscriptBlock[],
+ event: LiveTranscriptDelta,
+): LiveTranscriptBlock[] {
+ const delta = event.delta.trim();
+ if (!delta) return blocks;
+ const key = liveKey(event);
+ const existing = blocks.find((block) => block.key === key);
+ if (existing?.final) return blocks;
+ if (existing) {
+ return blocks.map((block) =>
+ block.key === key
+ ? {
+ ...block,
+ text: event.replace ? delta : `${block.text}${event.delta}`,
+ capturedAt: event.captured_at,
+ }
+ : block,
+ );
+ }
+
+ // Deepgram may revise the result start while an utterance is still open.
+ // One stream has one active partial: retire its stale key before appending
+ // the replacement so the realtime bubble cannot linger in the transcript.
+ return [
+ ...blocks.filter((block) => block.final || !sameLiveStream(block, event)),
+ {
+ key,
+ itemId: event.item_id,
+ deviceName: event.device_name,
+ deviceType: event.device_type,
+ provider: event.provider,
+ model: event.model,
+ text: delta,
+ capturedAt: event.captured_at,
+ final: false,
+ },
+ ];
+}
+
+export function applyLiveFinal(
+ blocks: LiveTranscriptBlock[],
+ event: LiveTranscriptFinal,
+): LiveTranscriptBlock[] {
+ const transcript = event.transcript.trim();
+ if (!transcript) return blocks;
+ const key = liveKey(event);
+ return [
+ ...blocks.filter(
+ (block) =>
+ block.key !== key && (block.final || !sameLiveStream(block, event)),
+ ),
+ {
+ key,
+ itemId: event.item_id,
+ deviceName: event.device_name,
+ deviceType: event.device_type,
+ speakerName: event.speaker_name,
+ provider: event.provider,
+ model: event.model,
+ text: transcript,
+ capturedAt: event.captured_at,
+ final: true,
+ },
+ ];
+}
+
function normalizeForDedupe(text: string) {
return text.toLowerCase().replace(/\s+/g, " ").trim();
}
@@ -174,6 +302,98 @@ function timestampMs(iso: string): number {
return Number.isFinite(ms) ? ms : 0;
}
+/**
+ * Keep provider-final live turns authoritative when the panel is reopened.
+ * Background STT remains available only for gaps outside the live coverage
+ * window; it must not replace a final with newly decoded words or speakers.
+ */
+export function filterBackgroundCoveredByLiveFinals(
+ chunks: MeetingAudioChunk[],
+ liveBlocks: LiveTranscriptBlock[],
+): MeetingAudioChunk[] {
+ const finals = liveBlocks
+ .filter((block) => block.final)
+ .map((block) => ({
+ timestamp: timestampMs(block.capturedAt),
+ deviceName: block.deviceName.trim().toLowerCase(),
+ deviceType: block.deviceType.trim().toLowerCase(),
+ }))
+ .filter((block) => block.timestamp > 0);
+ if (finals.length === 0) return chunks;
+
+ return chunks.filter((chunk) => {
+ if (chunk.source === "live") return true;
+ const timestamp = timestampMs(chunk.timestamp);
+ const deviceName = chunk.deviceName?.trim().toLowerCase() ?? "";
+ const deviceType = chunk.deviceType.trim().toLowerCase();
+ return !finals.some(
+ (final) =>
+ final.deviceType === deviceType &&
+ (!final.deviceName || !deviceName || final.deviceName === deviceName) &&
+ Math.abs(final.timestamp - timestamp) <= LIVE_AUTHORITY_WINDOW_MS,
+ );
+ });
+}
+
+/** Prefer clean system audio when the microphone hears the same nearby words. */
+export function filterLiveCrossDeviceEchoes(
+ chunks: MeetingAudioChunk[],
+ liveBlocks: LiveTranscriptBlock[],
+): LiveTranscriptBlock[] {
+ const echoWindowMs = 6_000;
+ const durableWindowMs = 15_000;
+ const durableBlocks = chunks
+ .map((chunk) => ({
+ timestamp: timestampMs(chunk.timestamp),
+ deviceName: chunk.deviceName?.trim().toLowerCase() ?? "",
+ deviceType: chunk.deviceType.trim().toLowerCase(),
+ text: normalizeForDedupe(chunk.transcription ?? ""),
+ }))
+ .filter((block) => block.timestamp > 0 && block.text.length > 0);
+ const outputBlocks = liveBlocks
+ .filter((block) => block.deviceType.toLowerCase() === "output")
+ .map((block) => ({
+ timestamp: timestampMs(block.capturedAt),
+ words: normalizeForDedupe(block.text).split(" ").filter(Boolean),
+ }));
+
+ return liveBlocks.filter((block) => {
+ const normalized = normalizeForDedupe(block.text);
+ const timestamp = timestampMs(block.capturedAt);
+ const deviceName = block.deviceName.trim().toLowerCase();
+ const deviceType = block.deviceType.trim().toLowerCase();
+ const alreadyDurable = durableBlocks.some(
+ (durable) =>
+ durable.deviceType === deviceType &&
+ (!durable.deviceName ||
+ !deviceName ||
+ durable.deviceName === deviceName) &&
+ Math.abs(durable.timestamp - timestamp) <= durableWindowMs &&
+ (durable.text.includes(normalized) ||
+ normalized.includes(durable.text)),
+ );
+ if (normalized && alreadyDurable) return false;
+
+ if (block.deviceType.toLowerCase() === "input") {
+ const nearbyOutputWords = new Set(
+ outputBlocks
+ .filter(
+ (output) => Math.abs(output.timestamp - timestamp) <= echoWindowMs,
+ )
+ .flatMap((output) => output.words),
+ );
+ const words = normalized.split(" ").filter(Boolean);
+ const covered =
+ words.length > 0
+ ? words.filter((word) => nearbyOutputWords.has(word)).length /
+ words.length
+ : 0;
+ if (covered >= 0.6) return false;
+ }
+ return true;
+ });
+}
+
function sortChunks(chunks: MeetingAudioChunk[]): MeetingAudioChunk[] {
return [...chunks].sort((a, b) => {
const byTime = timestampMs(a.timestamp) - timestampMs(b.timestamp);
@@ -203,9 +423,12 @@ function groupBySpeaker(chunks: MeetingAudioChunk[]): SpeakerBlock[] {
if (ts <= 0) continue;
const speakerName = c.speakerName || (c.isInput ? "me" : "speaker");
const speakerId = c.speakerId;
+ const speakerKey =
+ speakerId != null
+ ? `speaker:${speakerId}`
+ : `stream:${c.deviceName || c.deviceType}:${speakerName}`;
const last = out[out.length - 1];
- const sameSpeaker =
- last && last.speakerId === speakerId && last.speakerName === speakerName;
+ const sameSpeaker = last?.speakerKey === speakerKey;
// Glue if same speaker AND within 30s of last segment — keeps long pauses
// as paragraph breaks even when the same person is still talking.
if (sameSpeaker && ts - last.endMs < 30_000) {
@@ -217,6 +440,7 @@ function groupBySpeaker(chunks: MeetingAudioChunk[]): SpeakerBlock[] {
key: `${c.audioChunkId}-${ts}-${out.length}`,
speakerId,
speakerName,
+ speakerKey,
startMs: ts,
endMs: ts,
text,
@@ -231,7 +455,7 @@ function groupBySpeaker(chunks: MeetingAudioChunk[]): SpeakerBlock[] {
return out;
}
-function liveBlockToSpeakerBlock(
+export function liveBlockToSpeakerBlock(
block: LiveTranscriptBlock,
index: number,
): SpeakerBlock | null {
@@ -241,7 +465,14 @@ function liveBlockToSpeakerBlock(
return {
key: `live-${block.key}-${index}`,
speakerId: null,
- speakerName: block.deviceType.toLowerCase() === "input" ? "me" : "speaker",
+ speakerName:
+ block.speakerName?.trim() ||
+ (!block.final
+ ? "transcribing"
+ : block.deviceType.toLowerCase() === "input"
+ ? "me"
+ : "speaker"),
+ speakerKey: `stream:${block.deviceName}:${block.deviceType}:${block.speakerName?.trim() || "unknown"}`,
startMs,
endMs: startMs,
text,
@@ -308,7 +539,9 @@ export function TranscriptPanel({
const [query, setQuery] = useState("");
const [searchOpen, setSearchOpen] = useState(false);
const searchInputRef = useRef<HTMLInputElement | null>(null);
- const [liveBlocks, setLiveBlocks] = useState<LiveTranscriptBlock[]>([]);
+ const [liveBlocks, setLiveBlocks] = useState<LiveTranscriptBlock[]>(() =>
+ loadCachedLiveFinals(meeting.id),
+ );
const [liveStatus, setLiveStatus] = useState<LiveStreamingStatus | null>(
null,
);
@@ -434,7 +667,7 @@ export function TranscriptPanel({
}, [meeting.id, range.start, range.end, refreshKey]);
useEffect(() => {
- setLiveBlocks([]);
+ setLiveBlocks(loadCachedLiveFinals(meeting.id));
setLiveStatus(null);
setLiveError(null);
}, [meeting.id]);
@@ -451,34 +684,7 @@ export function TranscriptPanel({
const delta = event.payload.delta ?? "";
if (!delta.trim()) return;
setLiveError(null);
- const key = liveKey(event.payload);
- setLiveBlocks((prev) => {
- const existing = prev.find((b) => b.key === key);
- if (existing) {
- return prev.map((b) =>
- b.key === key
- ? {
- ...b,
- text: event.payload.replace ? delta : `${b.text}${delta}`,
- }
- : b,
- );
- }
- return [
- ...prev,
- {
- key,
- itemId: event.payload.item_id,
- deviceName: event.payload.device_name,
- deviceType: event.payload.device_type,
- provider: event.payload.provider,
- model: event.payload.model,
- text: delta,
- capturedAt: event.payload.captured_at,
- final: false,
- },
- ];
- });
+ setLiveBlocks((prev) => applyLiveDelta(prev, event.payload));
},
);
@@ -487,37 +693,12 @@ export function TranscriptPanel({
(event) => {
if (cancelled || Number(event.payload.meeting_id) !== meeting.id)
return;
- const transcript = (event.payload.transcript ?? "").trim();
- if (!transcript) return;
+ if (!(event.payload.transcript ?? "").trim()) return;
setLiveError(null);
- const key = liveKey(event.payload);
setLiveBlocks((prev) => {
- const existing = prev.find((b) => b.key === key);
- if (existing) {
- return prev.map((b) =>
- b.key === key
- ? {
- ...b,
- text: transcript,
- final: true,
- }
- : b,
- );
- }
- return [
- ...prev,
- {
- key,
- itemId: event.payload.item_id,
- deviceName: event.payload.device_name,
- deviceType: event.payload.device_type,
- provider: event.payload.provider,
- model: event.payload.model,
- text: transcript,
- capturedAt: event.payload.captured_at,
- final: true,
- },
- ];
+ const next = applyLiveFinal(prev, event.payload);
+ cacheLiveFinals(meeting.id, next);
+ return next;
});
},
);
@@ -586,51 +767,18 @@ export function TranscriptPanel({
};
}, [isOpen, meeting.id, range.start, range.end, isLive, refreshKey]);
- const blocks = useMemo(() => groupBySpeaker(chunks), [chunks]);
- const visibleLiveBlocks = useMemo(() => {
- const durableText = normalizeForDedupe(
- chunks.map((c) => c.transcription ?? "").join(" "),
- );
- // Cross-device echo suppression. Without headphones the mic ("input"/"me")
- // picks up the speaker output, so the remote's words arrive on BOTH the
- // input stream and the clean system-audio ("output"/"speaker") stream.
- // macOS VoiceProcessingIO AEC does not remove this (it has no downlink
- // reference), and the engine's cross-device dedup only runs on the deferred
- // durable path — so during a live meeting both copies reach the UI. The
- // output capture is the clean source, so drop an input block when most of
- // its words are covered by a nearby output block.
- const ECHO_WINDOW_MS = 6000;
- const outputBlocks = liveBlocks
- .filter((b) => b.deviceType.toLowerCase() === "output")
- .map((b) => ({
- ts: timestampMs(b.capturedAt),
- norm: normalizeForDedupe(b.text),
- }));
- const isInputEchoOfOutput = (
- block: LiveTranscriptBlock,
- normalized: string,
- ) => {
- if (block.deviceType.toLowerCase() !== "input") return false;
- const ts = timestampMs(block.capturedAt);
- const ref = new Set(
- outputBlocks
- .filter((o) => Math.abs(o.ts - ts) <= ECHO_WINDOW_MS)
- .flatMap((o) => o.norm.split(" "))
- .filter(Boolean),
- );
- if (ref.size === 0) return false;
- const words = normalized.split(" ").filter(Boolean);
- if (words.length === 0) return false;
- const covered = words.filter((w) => ref.has(w)).length / words.length;
- return covered >= 0.6;
- };
- return liveBlocks.filter((block) => {
- const normalized = normalizeForDedupe(block.text);
- if (normalized.length < 24) return true;
- if (durableText.includes(normalized.slice(0, 80))) return false;
- return !isInputEchoOfOutput(block, normalized);
- });
- }, [chunks, liveBlocks]);
+ const authoritativeChunks = useMemo(
+ () => filterBackgroundCoveredByLiveFinals(chunks, liveBlocks),
+ [chunks, liveBlocks],
+ );
+ const blocks = useMemo(
+ () => groupBySpeaker(authoritativeChunks),
+ [authoritativeChunks],
+ );
+ const visibleLiveBlocks = useMemo(
+ () => filterLiveCrossDeviceEchoes(authoritativeChunks, liveBlocks),
+ [authoritativeChunks, liveBlocks],
+ );
const visibleLiveSpeakerBlocks = useMemo(
() =>
visibleLiveBlocks
@@ -639,7 +787,10 @@ export function TranscriptPanel({
[visibleLiveBlocks],
);
const displayBlocks = useMemo(
- () => [...blocks, ...visibleLiveSpeakerBlocks].sort(compareBlocks),
+ () =>
+ coalesceFinalSpeakerRuns(
+ [...blocks, ...visibleLiveSpeakerBlocks].sort(compareBlocks),
+ ),
[blocks, visibleLiveSpeakerBlocks],
);
const latestBlockSignal = useMemo(() => {
@@ -962,7 +1113,11 @@ export function TranscriptPanel({
"h-7 w-7 p-0",
searchOpen && "bg-accent text-accent-foreground",
)}
- title={searchOpen ? "hide search" : `search transcript (${isMac ? "⌘F" : "Ctrl+F"})`}
+ title={
+ searchOpen
+ ? "hide search"
+ : `search transcript (${isMac ? "⌘F" : "Ctrl+F"})`
+ }
aria-label={
searchOpen ? "hide transcript search" : "search transcript"
}
@@ -1260,14 +1415,39 @@ export const SpeakerParagraph = React.memo(function SpeakerParagraph({
const SPEAKER_RUN_MAX_GAP_MS = 30_000;
+export function coalesceFinalSpeakerRuns(
+ blocks: SpeakerBlock[],
+): SpeakerBlock[] {
+ const merged: SpeakerBlock[] = [];
+ for (const block of blocks) {
+ const previous = merged[merged.length - 1];
+ if (
+ previous?.final &&
+ block.final &&
+ isSpeakerContinuation(previous, block)
+ ) {
+ previous.text = `${previous.text} ${block.text}`;
+ previous.endMs = Math.max(previous.endMs, block.endMs);
+ previous.segmentCount += block.segmentCount;
+ continue;
+ }
+ merged.push({ ...block });
+ }
+ return merged;
+}
+
export function isSpeakerContinuation(
previous: SpeakerBlock | undefined,
current: SpeakerBlock | undefined,
): boolean {
if (!previous || !current) return false;
+ const sameSpeaker =
+ previous.speakerKey && current.speakerKey
+ ? previous.speakerKey === current.speakerKey
+ : previous.speakerId === current.speakerId &&
+ previous.speakerName === current.speakerName;
return (
- previous.speakerId === current.speakerId &&
- previous.speakerName === current.speakerName &&
+ sameSpeaker &&
current.startMs >= previous.endMs &&
current.startMs - previous.endMs <= SPEAKER_RUN_MAX_GAP_MS
);
diff --git a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts b/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts
index 348697ffc..ac3b7db2f 100644
--- a/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts
+++ b/apps/screenpipe-app-tauri/lib/utils/meeting-context.ts
@@ -37,6 +37,7 @@ export interface MeetingAudioChunk {
audioFilePath: string;
speakerId: number | null;
speakerName: string;
+ deviceName?: string;
deviceType: string;
isInput: boolean;
transcription: string;
@@ -853,6 +854,7 @@ interface SearchAudioItem {
transcription?: string;
timestamp?: string;
file_path?: string;
+ device?: string;
device_type?: string;
speaker?: { id?: number; name?: string } | null;
};
@@ -904,6 +906,7 @@ export async function fetchMeetingAudio(
// Mic rows show "me" only until someone is actually assigned —
// hardcoding "me" made input lines look impossible to reassign.
speakerName: c.speaker?.name?.trim() || (isInput ? "me" : ""),
+ deviceName: c.device,
deviceType,
isInput,
transcription: c.transcription,
@@ -982,6 +985,7 @@ async function fetchRoutedMeetingTranscript(
// masked by a hardcoded "me".
speakerName:
segment.speakerName?.trim() || (isInput ? "me" : "speaker"),
+ deviceName: segment.deviceName,
deviceType,
isInput,
transcription: segment.transcript,
diff --git a/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rs b/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rs
index 8f2df47fb..3bb9954bc 100644
--- a/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rs
+++ b/crates/screenpipe-audio/src/meeting_streaming/deepgram_live.rs
@@ -366,19 +366,37 @@ fn handle_results_event(
let captured_at = latest_audio_time(latest_audio_ms);
if is_final {
- let speaker_name = resolve_speaker_name(value, device_type, config);
- let event = MeetingTranscriptFinal {
- meeting_id,
- provider: config.provider.as_str().to_string(),
- model: config.model.clone(),
- item_id,
- device_name: device_name.to_string(),
- device_type: device_type.to_string(),
- speaker_name,
- transcript: transcript.to_string(),
- captured_at,
- };
- let _ = screenpipe_events::send_event("meeting_transcript_final", event);
+ let turns = speaker_turns(value);
+ if turns.is_empty() {
+ emit_final(
+ meeting_id,
+ config,
+ device_name,
+ device_type,
+ item_id,
+ None,
+ transcript.to_string(),
+ captured_at,
+ );
+ } else {
+ for (index, turn) in turns.into_iter().enumerate() {
+ let turn_item_id = if index == 0 {
+ item_id.clone()
+ } else {
+ format!("{item_id}:turn:{index}")
+ };
+ emit_final(
+ meeting_id,
+ config,
+ device_name,
+ device_type,
+ turn_item_id,
+ Some(format!("speaker {}", turn.speaker + 1)),
+ turn.transcript,
+ captured_at + chrono::Duration::milliseconds(index as i64),
+ );
+ }
+ }
} else {
let event = MeetingTranscriptDelta {
meeting_id,
@@ -395,34 +413,74 @@ fn handle_results_event(
}
}
-/// Pick a speaker label from a Deepgram `Results` payload with `diarize=true`.
-/// Counts speaker indices across the words array and returns the dominant one.
-/// `device_type == "input"` + `speaker 0` is treated as the local user when
-/// `local_speaker_name` is set, since deepgram numbers speakers in order of
-/// first utterance and the local user typically speaks first into their own
-/// mic. All other indices fall back to a generic `speaker N` (1-indexed).
-fn resolve_speaker_name(
- value: &Value,
- device_type: &str,
+fn emit_final(
+ meeting_id: i64,
config: &MeetingStreamingConfig,
-) -> Option<String> {
- let words = value
+ device_name: &str,
+ device_type: &str,
+ item_id: String,
+ speaker_name: Option<String>,
+ transcript: String,
+ captured_at: DateTime<Utc>,
+) {
+ let event = MeetingTranscriptFinal {
+ meeting_id,
+ provider: config.provider.as_str().to_string(),
+ model: config.model.clone(),
+ item_id,
+ device_name: device_name.to_string(),
+ device_type: device_type.to_string(),
+ speaker_name,
+ transcript,
+ captured_at,
+ };
+ let _ = screenpipe_events::send_event("meeting_transcript_final", event);
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct DeepgramSpeakerTurn {
+ speaker: i64,
+ transcript: String,
+}
+
+/// Preserve Deepgram's consecutive per-word speaker runs instead of assigning
+/// a mixed-speaker result to whichever speaker happened to say more words.
+///
+/// A partial diarization payload falls back to the top-level transcript rather
+/// than dropping unlabelled words or guessing their speaker.
+fn speaker_turns(value: &Value) -> Vec<DeepgramSpeakerTurn> {
+ let Some(words) = value
.pointer("/channel/alternatives/0/words")
- .and_then(Value::as_array)?;
- let mut counts: std::collections::HashMap<i64, usize> = std::collections::HashMap::new();
+ .and_then(Value::as_array)
+ else {
+ return Vec::new();
+ };
+ let mut turns: Vec<DeepgramSpeakerTurn> = Vec::new();
for word in words {
- if let Some(speaker) = word.get("speaker").and_then(Value::as_i64) {
- *counts.entry(speaker).or_insert(0) += 1;
- }
- }
- let (dominant, _) = counts.into_iter().max_by_key(|&(_, n)| n)?;
+ let Some(text) = word
+ .get("punctuated_word")
+ .and_then(Value::as_str)
+ .or_else(|| word.get("word").and_then(Value::as_str))
+ .map(str::trim)
+ .filter(|text| !text.is_empty())
+ else {
+ continue;
+ };
+ let Some(speaker) = word.get("speaker").and_then(Value::as_i64) else {
+ return Vec::new();
+ };
- if dominant == 0 && device_type == "input" {
- if let Some(name) = config.local_speaker_name.clone() {
- return Some(name);
+ if let Some(turn) = turns.last_mut().filter(|turn| turn.speaker == speaker) {
+ turn.transcript.push(' ');
+ turn.transcript.push_str(text);
+ } else {
+ turns.push(DeepgramSpeakerTurn {
+ speaker,
+ transcript: text.to_string(),
+ });
}
}
- Some(format!("speaker {}", dominant + 1))
+ turns
}
fn latest_audio_time(latest_audio_ms: &AtomicU64) -> DateTime<Utc> {
@@ -476,6 +534,48 @@ fn device_type_label(device_type: &DeviceType) -> &'static str {
mod tests {
use super::*;
+ #[test]
+ fn preserves_consecutive_deepgram_speaker_turns() {
+ let payload = json!({
+ "channel": { "alternatives": [{ "words": [
+ { "word": "hello", "punctuated_word": "Hello,", "speaker": 0 },
+ { "word": "there", "punctuated_word": "there.", "speaker": 0 },
+ { "word": "hi", "punctuated_word": "Hi!", "speaker": 1 },
+ { "word": "yes", "punctuated_word": "Yes.", "speaker": 0 }
+ ]}]}
+ });
+
+ assert_eq!(
+ speaker_turns(&payload),
+ vec![
+ DeepgramSpeakerTurn {
+ speaker: 0,
+ transcript: "Hello, there.".to_string(),
+ },
+ DeepgramSpeakerTurn {
+ speaker: 1,
+ transcript: "Hi!".to_string(),
+ },
+ DeepgramSpeakerTurn {
+ speaker: 0,
+ transcript: "Yes.".to_string(),
+ },
+ ]
+ );
+ }
+
+ #[test]
+ fn incomplete_diarization_falls_back_without_dropping_words() {
+ let payload = json!({
+ "channel": { "alternatives": [{ "words": [
+ { "word": "labelled", "speaker": 0 },
+ { "word": "unknown" }
+ ]}]}
+ });
+
+ assert!(speaker_turns(&payload).is_empty());
+ }
+
fn live_query(language: Option<&str>) -> String {
let config = MeetingStreamingConfig {
enabled: true,
diff --git a/crates/screenpipe-db/src/db/meetings.rs b/crates/screenpipe-db/src/db/meetings.rs
index b89dc169d..1fbc0b1e0 100644
--- a/crates/screenpipe-db/src/db/meetings.rs
+++ b/crates/screenpipe-db/src/db/meetings.rs
@@ -1184,15 +1184,11 @@ impl DatabaseManager {
NULL AS audio_chunk_id,
NULL AS audio_file_path,
mts.speaker_id AS speaker_id,
- -- Prefer the resolved global speaker's name; output rows fall
- -- back to the free-text Deepgram label until backfilled / if
- -- the speaker is unnamed (NULLIF treats '' as "no name yet").
- -- Input (mic) rows get NULL instead of the raw label so the
- -- client can render "me" until the user assigns someone.
- CASE
- WHEN mts.device_type = 'input' THEN NULLIF(s.name, '')
- ELSE COALESCE(NULLIF(s.name, ''), mts.speaker_name)
- END AS speaker_name,
+ -- Prefer the resolved global speaker's name, then preserve
+ -- Deepgram's stream-local label until voice backfill gives it
+ -- a durable identity. A mic is a capture source, not proof
+ -- that every voice on it belongs to the local user.
+ COALESCE(NULLIF(s.name, ''), mts.speaker_name) AS speaker_name,
mts.transcript,
mts.captured_at,
mts.created_at
diff --git a/crates/screenpipe-db/tests/timeline_live_meeting_test.rs b/crates/screenpipe-db/tests/timeline_live_meeting_test.rs
index 625912a2a..9dd920708 100644
--- a/crates/screenpipe-db/tests/timeline_live_meeting_test.rs
+++ b/crates/screenpipe-db/tests/timeline_live_meeting_test.rs
@@ -499,9 +499,10 @@ mod timeline_live_meeting_tests {
}
/// Until a segment is resolved, the Meeting view falls back to Deepgram's
- /// free-text `speaker_name`, and the backfill maps nothing.
+ /// free-text `speaker_name` even for mic input. A capture source is not a
+ /// speaker identity, and the backfill may have nothing to map yet.
#[tokio::test]
- async fn test_meeting_segment_falls_back_to_freetext_speaker() {
+ async fn test_input_meeting_segment_falls_back_to_freetext_speaker() {
let db = setup_test_db().await;
let base = Utc::now();
@@ -514,8 +515,8 @@ mod timeline_live_meeting_tests {
"screenpipe-cloud",
None,
"deepgram:0:0",
- "System Audio",
- "output",
+ "Built-in Mic",
+ "input",
Some("speaker 2"),
"unresolved line",
base,