Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat(notes): add speaker recognition for meeting transcripts

Gabriel Stein616576fb906c
diff --git a/preload.js b/preload.js
index 8f48dcce4..a297c95cd 100644
--- a/preload.js
+++ b/preload.js
@@ -263,6 +263,16 @@ contextBridge.exposeInMainWorld("electronAPI", {
(callback) => (_event, data) => callback(data)
),
+ // Speaker name mapping
+ getSpeakerMappings: (noteId) => ipcRenderer.invoke("get-speaker-mappings", noteId),
+ setSpeakerMapping: (noteId, speakerId, displayName, email, profileId) =>
+ ipcRenderer.invoke("set-speaker-mapping", noteId, speakerId, displayName, email, profileId),
+ removeSpeakerMapping: (noteId, speakerId) =>
+ ipcRenderer.invoke("remove-speaker-mapping", noteId, speakerId),
+ getSpeakerProfiles: () => ipcRenderer.invoke("get-speaker-profiles"),
+ saveNoteSpeakerEmbeddings: (noteId, embeddings) =>
+ ipcRenderer.invoke("save-note-speaker-embeddings", noteId, embeddings),
+
// Window control functions
windowMinimize: () => ipcRenderer.invoke("window-minimize"),
windowMaximize: () => ipcRenderer.invoke("window-maximize"),
diff --git a/src/components/notes/MeetingTranscriptChat.tsx b/src/components/notes/MeetingTranscriptChat.tsx
index 5ee6a292c..510517593 100644
--- a/src/components/notes/MeetingTranscriptChat.tsx
+++ b/src/components/notes/MeetingTranscriptChat.tsx
@@ -1,5 +1,7 @@
-import { useEffect, useRef } from "react";
+import { useState, useEffect, useRef } from "react";
import { useTranslation } from "react-i18next";
+import { Check, X } from "lucide-react";
+import { Popover, PopoverTrigger, PopoverContent } from "../ui/popover";
import { cn } from "../lib/utils";
import type { TranscriptSegment } from "../../hooks/useMeetingTranscription";
@@ -72,16 +74,209 @@ function PartialBubble({ text, source }: { text: string; source: "mic" | "system
);
}
+function SpeakerLabel({
+ speakerId,
+ segment,
+ mappedName,
+ speakerProfiles,
+ participants,
+ colorIdx,
+ isYou,
+ onMap,
+ onConfirm,
+ onDismiss,
+ t,
+}: {
+ speakerId: string;
+ segment: TranscriptSegment;
+ mappedName?: string;
+ speakerProfiles?: Array<{ id: number; display_name: string; email: string | null }>;
+ participants?: Array<{ email: string; displayName: string | null }>;
+ colorIdx: number;
+ isYou: boolean;
+ onMap?: (speakerId: string, name: string, email?: string | null, profileId?: number) => void;
+ onConfirm?: (speakerId: string, name: string, profileId: number) => void;
+ onDismiss?: (speakerId: string) => void;
+ t: (key: string, opts?: Record<string, unknown>) => string;
+}) {
+ const [open, setOpen] = useState(false);
+ const [search, setSearch] = useState("");
+
+ if (isYou) {
+ return (
+ <span className="text-[11px] font-medium mb-0.5 px-1 text-primary/60">
+ {t("notes.speaker.you")}
+ </span>
+ );
+ }
+
+ const hasSuggestion = !!segment.suggestedName && !mappedName;
+
+ if (hasSuggestion) {
+ return (
+ <span className="group inline-flex items-center gap-1 mb-0.5 px-1">
+ <span className="text-[11px] font-medium italic text-muted-foreground/60">
+ {segment.suggestedName}
+ </span>
+ <button
+ onClick={() =>
+ onConfirm?.(speakerId, segment.suggestedName!, segment.suggestedProfileId!)
+ }
+ className="opacity-0 group-hover:opacity-100 p-0.5 rounded transition-opacity cursor-pointer text-muted-foreground hover:text-emerald-500"
+ >
+ <Check size={12} />
+ </button>
+ <button
+ onClick={() => onDismiss?.(speakerId)}
+ className="opacity-0 group-hover:opacity-100 p-0.5 rounded transition-opacity cursor-pointer text-muted-foreground hover:text-destructive"
+ >
+ <X size={12} />
+ </button>
+ </span>
+ );
+ }
+
+ const speakerNumber = (() => {
+ const match = speakerId.match(/speaker_(\d+)/);
+ return match ? Number(match[1]) + 1 : 1;
+ })();
+
+ const displayLabel = mappedName || segment.speakerName || t("notes.speaker.label", { n: speakerNumber });
+ const isUnmapped = !mappedName && !segment.speakerName;
+
+ const filteredParticipants = participants?.filter(
+ (p) =>
+ search === "" ||
+ (p.displayName || p.email).toLowerCase().includes(search.toLowerCase())
+ );
+
+ const filteredProfiles = speakerProfiles?.filter(
+ (p) =>
+ search === "" ||
+ p.display_name.toLowerCase().includes(search.toLowerCase()) ||
+ (p.email && p.email.toLowerCase().includes(search.toLowerCase()))
+ );
+
+ const handleSelect = (name: string, email?: string | null, profileId?: number) => {
+ onMap?.(speakerId, name, email, profileId);
+ setOpen(false);
+ setSearch("");
+ };
+
+ const handleKeyDown = (e: React.KeyboardEvent) => {
+ if (e.key === "Enter" && search.trim()) {
+ e.preventDefault();
+ handleSelect(search.trim());
+ }
+ };
+
+ return (
+ <Popover
+ open={open}
+ onOpenChange={(o) => {
+ setOpen(o);
+ if (!o) setSearch("");
+ }}
+ >
+ <PopoverTrigger asChild>
+ <button
+ className={cn(
+ "text-[11px] font-medium mb-0.5 px-1 transition-colors duration-100 cursor-pointer rounded-sm outline-none focus-visible:ring-1 focus-visible:ring-ring",
+ SPEAKER_COLORS[colorIdx],
+ isUnmapped && "border-b border-dotted border-current",
+ "hover:bg-muted/50"
+ )}
+ >
+ {displayLabel}
+ </button>
+ </PopoverTrigger>
+ <PopoverContent className="w-72 p-0">
+ <div className="p-2 border-b border-border/50">
+ <input
+ value={search}
+ onChange={(e) => setSearch(e.target.value)}
+ onKeyDown={handleKeyDown}
+ placeholder={t("notes.speaker.typeNamePlaceholder")}
+ className="w-full px-2 py-1.5 rounded-md bg-transparent text-xs text-foreground placeholder:text-foreground/20 outline-none border-none appearance-none"
+ autoFocus
+ />
+ </div>
+ <div className="max-h-40 overflow-y-auto">
+ {filteredParticipants && filteredParticipants.length > 0 && (
+ <div className="p-1 border-b border-border/30">
+ <div className="px-2 py-1 text-[11px] font-medium text-muted-foreground">
+ {t("notes.speaker.meetingAttendees")}
+ </div>
+ {filteredParticipants.slice(0, 5).map((p) => (
+ <button
+ key={p.email}
+ onClick={() => handleSelect(p.displayName || p.email.split("@")[0], p.email)}
+ className="flex items-center gap-2 w-full px-2 py-1.5 rounded-md text-xs text-foreground/70 hover:bg-foreground/5 transition-colors cursor-pointer"
+ >
+ <span className="truncate">{p.displayName || p.email}</span>
+ </button>
+ ))}
+ </div>
+ )}
+ {filteredProfiles && filteredProfiles.length > 0 && (
+ <div className="p-1">
+ <div className="px-2 py-1 text-[11px] font-medium text-muted-foreground">
+ {t("notes.speaker.knownSpeakers")}
+ </div>
+ {filteredProfiles.slice(0, 5).map((p) => (
+ <button
+ key={p.id}
+ onClick={() => handleSelect(p.display_name, p.email, p.id)}
+ className="flex items-center gap-2 w-full px-2 py-1.5 rounded-md text-xs text-foreground/70 hover:bg-foreground/5 transition-colors cursor-pointer"
+ >
+ <span className="truncate">{p.display_name}</span>
+ {p.email && (
+ <span className="text-foreground/30 truncate text-[11px]">{p.email}</span>
+ )}
+ </button>
+ ))}
+ </div>
+ )}
+ {(!filteredParticipants || filteredParticipants.length === 0) &&
+ (!filteredProfiles || filteredProfiles.length === 0) &&
+ !search && (
+ <div className="px-3 py-4 text-center text-[11px] text-foreground/30">
+ {t("notes.speaker.typeNamePlaceholder")}
+ </div>
+ )}
+ </div>
+ </PopoverContent>
+ </Popover>
+ );
+}
+
interface MeetingTranscriptChatProps {
segments: TranscriptSegment[];
micPartial?: string;
systemPartial?: string;
+ speakerMappings?: Record<string, string>;
+ speakerProfiles?: Array<{ id: number; display_name: string; email: string | null }>;
+ participants?: Array<{ email: string; displayName: string | null }>;
+ onMapSpeaker?: (
+ speakerId: string,
+ displayName: string,
+ email?: string | null,
+ profileId?: number
+ ) => void;
+ onConfirmSuggestion?: (speakerId: string, suggestedName: string, profileId: number) => void;
+ onDismissSuggestion?: (speakerId: string) => void;
}
export function MeetingTranscriptChat({
segments,
micPartial,
systemPartial,
+ speakerMappings,
+ speakerProfiles,
+ participants,
+ onMapSpeaker,
+ onConfirmSuggestion,
+ onDismissSuggestion,
}: MeetingTranscriptChatProps) {
const { t } = useTranslation();
const scrollRef = useRef<HTMLDivElement>(null);
@@ -89,7 +284,6 @@ export function MeetingTranscriptChat({
useEffect(() => {
const el = scrollRef.current;
if (!el) return;
- // Auto-scroll only when near the bottom (within 80px)
const isNearBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 80;
if (isNearBottom) {
el.scrollTop = el.scrollHeight;
@@ -125,17 +319,6 @@ export function MeetingTranscriptChat({
const isSystemSpeaker = hasSpeaker && !isYou;
const colorIdx = isSystemSpeaker ? getSpeakerColorIndex(segment.speaker!) : 0;
- let speakerLabel: string | null = null;
- if (hasSpeaker && !sameSpeaker) {
- if (isYou) {
- speakerLabel = t("notes.speaker.you");
- } else {
- const match = segment.speaker!.match(/speaker_(\d+)/);
- const n = match ? Number(match[1]) + 1 : 1;
- speakerLabel = t("notes.speaker.label", { n });
- }
- }
-
return (
<div
key={segment.id}
@@ -146,15 +329,20 @@ export function MeetingTranscriptChat({
)}
style={{ animation: "agent-message-in 200ms ease-out both" }}
>
- {speakerLabel && (
- <span
- className={cn(
- "text-[11px] font-medium mb-0.5 px-1",
- isYou ? "text-primary/60" : SPEAKER_COLORS[colorIdx]
- )}
- >
- {speakerLabel}
- </span>
+ {hasSpeaker && !sameSpeaker && (
+ <SpeakerLabel
+ speakerId={segment.speaker!}
+ segment={segment}
+ mappedName={speakerMappings?.[segment.speaker!]}
+ speakerProfiles={speakerProfiles}
+ participants={participants}
+ colorIdx={colorIdx}
+ isYou={isYou}
+ onMap={onMapSpeaker}
+ onConfirm={onConfirmSuggestion}
+ onDismiss={onDismissSuggestion}
+ t={t}
+ />
)}
<div
className={cn(
diff --git a/src/components/notes/NoteEditor.tsx b/src/components/notes/NoteEditor.tsx
index 37e591426..60ee6ef9b 100644
--- a/src/components/notes/NoteEditor.tsx
+++ b/src/components/notes/NoteEditor.tsx
@@ -37,6 +37,20 @@ import { parseTranscriptSegments } from "../../utils/parseTranscriptSegments";
import NoteParticipants from "./NoteParticipants";
import type { CalendarAttendee } from "../../types/calendar";
+function serializeTranscript(segments: TranscriptSegment[]): string {
+ return JSON.stringify(
+ segments.map((s) => ({
+ text: s.text,
+ source: s.source,
+ timestamp: s.timestamp,
+ speaker: s.speaker,
+ speakerName: s.speakerName,
+ suggestedName: s.suggestedName,
+ suggestedProfileId: s.suggestedProfileId,
+ }))
+ );
+}
+
function formatNoteDate(dateStr: string): string {
const date = normalizeDbDate(dateStr);
if (Number.isNaN(date.getTime())) return "";
@@ -128,6 +142,10 @@ export default function NoteEditor({
const [newFolderName, setNewFolderName] = useState("");
const [isDiarizing, setIsDiarizing] = useState(false);
const [diarizedSegments, setDiarizedSegments] = useState<TranscriptSegment[] | null>(null);
+ const [speakerMappings, setSpeakerMappings] = useState<Record<string, string>>({});
+ const [speakerProfiles, setSpeakerProfiles] = useState<
+ Array<{ id: number; display_name: string; email: string | null }>
+ >([]);
const editorRef = useRef<Editor | null>(null);
const embeddedChat = useEmbeddedChat({
@@ -214,6 +232,7 @@ export default function NoteEditor({
setChatMode("hidden");
setDiarizedSegments(null);
setIsDiarizing(false);
+ setSpeakerMappings({});
if (!isMeetingRecording) {
setViewMode("raw");
}
@@ -224,6 +243,19 @@ export default function NoteEditor({
}
}, [note.id, isMeetingRecording]);
+ useEffect(() => {
+ window.electronAPI?.getSpeakerMappings?.(note.id).then((mappings) => {
+ const map: Record<string, string> = {};
+ for (const m of mappings || []) map[m.speaker_id] = m.display_name;
+ setSpeakerMappings(map);
+ });
+ window.electronAPI?.getSpeakerProfiles?.().then((profiles) => {
+ setSpeakerProfiles(
+ (profiles || []).map((p) => ({ id: p.id, display_name: p.display_name, email: p.email }))
+ );
+ });
+ }, [note.id]);
+
// Auto-show chat when hook loads an existing conversation with messages
useEffect(() => {
if (
@@ -265,19 +297,86 @@ export default function NoteEditor({
}));
setDiarizedSegments(enriched);
- const serialized = JSON.stringify(
- enriched.map((s: TranscriptSegment) => ({
- text: s.text,
- source: s.source,
- timestamp: s.timestamp,
- speaker: s.speaker,
- }))
- );
- window.electronAPI.updateNote(note.id, { transcript: serialized });
+ window.electronAPI.updateNote(note.id, { transcript: serializeTranscript(enriched) });
+
+ if (data.speakerEmbeddings) {
+ window.electronAPI?.saveNoteSpeakerEmbeddings?.(note.id, data.speakerEmbeddings);
+ }
+
+ const autoMappings: Record<string, string> = {};
+ for (const s of enriched) {
+ if (s.speakerName && s.speaker) autoMappings[s.speaker] = s.speakerName;
+ }
+ if (Object.keys(autoMappings).length > 0) {
+ setSpeakerMappings((prev) => ({ ...prev, ...autoMappings }));
+ }
});
return () => cleanup?.();
}, [note.id, diarizationSessionId]);
+ const handleMapSpeaker = useCallback(
+ async (
+ speakerId: string,
+ displayName: string,
+ email?: string | null,
+ profileId?: number | null
+ ) => {
+ setSpeakerMappings((prev) => ({ ...prev, [speakerId]: displayName }));
+ await window.electronAPI?.setSpeakerMapping?.(
+ note.id,
+ speakerId,
+ displayName,
+ email,
+ profileId
+ );
+
+ const currentSegments = displaySegments.map((s) =>
+ s.speaker === speakerId
+ ? {
+ ...s,
+ speakerName: displayName,
+ suggestedName: undefined,
+ suggestedProfileId: undefined,
+ }
+ : s
+ );
+ window.electronAPI?.updateNote(note.id, { transcript: serializeTranscript(currentSegments) });
+
+ window.electronAPI?.getSpeakerProfiles?.().then((profiles) => {
+ setSpeakerProfiles(
+ (profiles || []).map((p) => ({
+ id: p.id,
+ display_name: p.display_name,
+ email: p.email,
+ }))
+ );
+ });
+ },
+ [note.id, displaySegments]
+ );
+
+ const handleConfirmSuggestion = useCallback(
+ async (speakerId: string, suggestedName: string, profileId: number) => {
+ await handleMapSpeaker(speakerId, suggestedName, null, profileId);
+ },
+ [handleMapSpeaker]
+ );
+
+ const handleDismissSuggestion = useCallback(
+ async (speakerId: string) => {
+ const currentSegments = displaySegments.map((s) =>
+ s.speaker === speakerId
+ ? { ...s, suggestedName: undefined, suggestedProfileId: undefined }
+ : s
+ );
+ if (diarizedSegments) {
+ setDiarizedSegments(currentSegments);
+ }
+ window.electronAPI?.updateNote(note.id, { transcript: serializeTranscript(currentSegments) });
+ },
+ [note.id, displaySegments, diarizedSegments]
+ );
+
const handleTitleInput = useCallback(() => {
if (titleRef.current) {
const text = titleRef.current.textContent || "";
@@ -587,6 +686,12 @@ export default function NoteEditor({
segments={displaySegments}
micPartial={isMeetingRecording ? meetingMicPartial : undefined}
systemPartial={isMeetingRecording ? meetingSystemPartial : undefined}
+ speakerMappings={speakerMappings}
+ speakerProfiles={speakerProfiles}
+ participants={parsedParticipants}
+ onMapSpeaker={handleMapSpeaker}
+ onConfirmSuggestion={handleConfirmSuggestion}
+ onDismissSuggestion={handleDismissSuggestion}
/>
{isDiarizing && (
<div className="flex items-center justify-center gap-1.5 py-2">
diff --git a/src/helpers/database.js b/src/helpers/database.js
index 52f407528..63fa903e0 100644
--- a/src/helpers/database.js
+++ b/src/helpers/database.js
@@ -383,6 +383,40 @@ class DatabaseManager {
)
`);
+ this.db.exec(`
+ CREATE TABLE IF NOT EXISTS speaker_profiles (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ display_name TEXT NOT NULL,
+ email TEXT,
+ embedding BLOB NOT NULL,
+ sample_count INTEGER DEFAULT 1,
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ `);
+
+ this.db.exec(`
+ CREATE TABLE IF NOT EXISTS speaker_mappings (
+ note_id INTEGER NOT NULL,
+ speaker_id TEXT NOT NULL,
+ profile_id INTEGER,
+ display_name TEXT NOT NULL,
+ PRIMARY KEY (note_id, speaker_id),
+ FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE CASCADE,
+ FOREIGN KEY (profile_id) REFERENCES speaker_profiles(id) ON DELETE SET NULL
+ )
+ `);
+
+ this.db.exec(`
+ CREATE TABLE IF NOT EXISTS note_speaker_embeddings (
+ note_id INTEGER NOT NULL,
+ speaker_id TEXT NOT NULL,
+ embedding BLOB NOT NULL,
+ PRIMARY KEY (note_id, speaker_id),
+ FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE CASCADE
+ )
+ `);
+
return true;
} catch (error) {
debugLogger.error("Database initialization failed", { error: error.message }, "database");
@@ -1482,6 +1516,164 @@ class DatabaseManager {
throw error;
}
}
+
+ upsertSpeakerProfile(name, email, embeddingBuffer, profileId = null) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ const existing = profileId
+ ? this.db.prepare("SELECT * FROM speaker_profiles WHERE id = ?").get(profileId)
+ : this.db.prepare("SELECT * FROM speaker_profiles WHERE display_name = ?").get(name);
+ if (existing) {
+ const stored = new Float32Array(
+ existing.embedding.buffer,
+ existing.embedding.byteOffset,
+ existing.embedding.byteLength / 4
+ );
+ const incoming = new Float32Array(
+ embeddingBuffer.buffer,
+ embeddingBuffer.byteOffset,
+ embeddingBuffer.byteLength / 4
+ );
+ const updated = new Float32Array(stored.length);
+ for (let i = 0; i < stored.length; i++) {
+ updated[i] = 0.3 * incoming[i] + 0.7 * stored[i];
+ }
+ const updatedBuf = Buffer.from(updated.buffer);
+ this.db
+ .prepare(
+ "UPDATE speaker_profiles SET display_name = ?, email = ?, embedding = ?, sample_count = sample_count + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?"
+ )
+ .run(name, email || existing.email || null, updatedBuf, existing.id);
+ return this.db.prepare("SELECT * FROM speaker_profiles WHERE id = ?").get(existing.id);
+ }
+ const result = this.db
+ .prepare(
+ "INSERT INTO speaker_profiles (display_name, email, embedding) VALUES (?, ?, ?)"
+ )
+ .run(name, email || null, embeddingBuffer);
+ return this.db
+ .prepare("SELECT * FROM speaker_profiles WHERE id = ?")
+ .get(result.lastInsertRowid);
+ } catch (error) {
+ debugLogger.error("Error upserting speaker profile", { error: error.message }, "database");
+ throw error;
+ }
+ }
+
+ getSpeakerProfiles(includeEmbedding = false) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ const query = includeEmbedding
+ ? "SELECT * FROM speaker_profiles"
+ : `SELECT id, display_name, email, sample_count, created_at, updated_at
+ FROM speaker_profiles`;
+ return this.db.prepare(query).all();
+ } catch (error) {
+ debugLogger.error("Error getting speaker profiles", { error: error.message }, "database");
+ throw error;
+ }
+ }
+
+ setSpeakerMapping(noteId, speakerId, profileId, displayName) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ this.db
+ .prepare(
+ "INSERT OR REPLACE INTO speaker_mappings (note_id, speaker_id, profile_id, display_name) VALUES (?, ?, ?, ?)"
+ )
+ .run(noteId, speakerId, profileId, displayName);
+ return { success: true };
+ } catch (error) {
+ debugLogger.error("Error setting speaker mapping", { error: error.message }, "database");
+ throw error;
+ }
+ }
+
+ getSpeakerMappings(noteId) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ return this.db
+ .prepare("SELECT * FROM speaker_mappings WHERE note_id = ?")
+ .all(noteId);
+ } catch (error) {
+ debugLogger.error("Error getting speaker mappings", { error: error.message }, "database");
+ throw error;
+ }
+ }
+
+ saveNoteSpeakerEmbeddings(noteId, embeddings) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ const transaction = this.db.transaction((entries) => {
+ const stmt = this.db.prepare(
+ "INSERT OR REPLACE INTO note_speaker_embeddings (note_id, speaker_id, embedding) VALUES (?, ?, ?)"
+ );
+ for (const [speakerId, buffer] of entries) {
+ stmt.run(noteId, speakerId, buffer);
+ }
+ });
+ transaction(Object.entries(embeddings));
+ return { success: true };
+ } catch (error) {
+ debugLogger.error(
+ "Error saving note speaker embeddings",
+ { error: error.message },
+ "database"
+ );
+ throw error;
+ }
+ }
+
+ getNoteSpeakerEmbeddings(noteId) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ return this.db
+ .prepare("SELECT * FROM note_speaker_embeddings WHERE note_id = ?")
+ .all(noteId);
+ } catch (error) {
+ debugLogger.error(
+ "Error getting note speaker embeddings",
+ { error: error.message },
+ "database"
+ );
+ throw error;
+ }
+ }
+
+ getNotesWithUnmappedSpeakers() {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ return this.db
+ .prepare(
+ `SELECT DISTINCT nse.note_id
+ FROM note_speaker_embeddings nse
+ LEFT JOIN speaker_mappings sm ON nse.note_id = sm.note_id AND nse.speaker_id = sm.speaker_id
+ WHERE sm.note_id IS NULL`
+ )
+ .all()
+ .map((row) => row.note_id);
+ } catch (error) {
+ debugLogger.error(
+ "Error getting notes with unmapped speakers",
+ { error: error.message },
+ "database"
+ );
+ throw error;
+ }
+ }
+
+ removeSpeakerMapping(noteId, speakerId) {
+ try {
+ if (!this.db) throw new Error("Database not initialized");
+ this.db
+ .prepare("DELETE FROM speaker_mappings WHERE note_id = ? AND speaker_id = ?")
+ .run(noteId, speakerId);
+ return { success: true };
+ } catch (error) {
+ debugLogger.error("Error removing speaker mapping", { error: error.message }, "database");
+ throw error;
+ }
+ }
}
module.exports = DatabaseManager;
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 66ef6a9b2..91453cccf 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -5084,6 +5084,97 @@ class IPCHandlers {
return { canceled: true };
}
});
+
+ ipcMain.handle("get-speaker-mappings", async (_event, noteId) => {
+ return this.databaseManager.getSpeakerMappings(noteId);
+ });
+
+ ipcMain.handle(
+ "set-speaker-mapping",
+ async (_event, noteId, speakerId, displayName, email, profileId) => {
+ const embeddings = this.databaseManager.getNoteSpeakerEmbeddings(noteId);
+ const speakerEmb = embeddings.find((e) => e.speaker_id === speakerId);
+
+ let resolvedProfileId = profileId ?? null;
+ if (speakerEmb) {
+ const profile = this.databaseManager.upsertSpeakerProfile(
+ displayName,
+ email || null,
+ speakerEmb.embedding,
+ resolvedProfileId
+ );
+ resolvedProfileId = profile.id;
+ this._retroactiveMapping(profile);
+ }
+
+ this.databaseManager.setSpeakerMapping(noteId, speakerId, resolvedProfileId, displayName);
+ return { success: true, profileId: resolvedProfileId };
+ }
+ );
+
+ ipcMain.handle("remove-speaker-mapping", async (_event, noteId, speakerId) => {
+ this.databaseManager.removeSpeakerMapping(noteId, speakerId);
+ return { success: true };
+ });
+
+ ipcMain.handle("get-speaker-profiles", async () => {
+ return this.databaseManager.getSpeakerProfiles();
+ });
+
+ ipcMain.handle("save-note-speaker-embeddings", async (_event, noteId, embeddingsObj) => {
+ const buffers = {};
+ for (const [speakerId, arr] of Object.entries(embeddingsObj)) {
+ buffers[speakerId] = Buffer.from(new Float32Array(arr).buffer);
+ }
+ this.databaseManager.saveNoteSpeakerEmbeddings(noteId, buffers);
+ return { success: true };
+ });
+ }
+
+ _retroactiveMapping(profile) {
+ setImmediate(async () => {
+ try {
+ const speakerEmbeddings = require("./speakerEmbeddings");
+ const noteIds = this.databaseManager.getNotesWithUnmappedSpeakers();
+
+ const profileEmb = new Float32Array(profile.embedding.buffer, profile.embedding.byteOffset, profile.embedding.byteLength / 4);
+
+ for (const noteId of noteIds) {
+ const embeddings = this.databaseManager.getNoteSpeakerEmbeddings(noteId);
+ const existing = this.databaseManager.getSpeakerMappings(noteId);
+ const mappedSpeakers = new Set(existing.map(m => m.speaker_id));
+ for (const emb of embeddings) {
+ if (mappedSpeakers.has(emb.speaker_id)) continue;
+
+ const speakerEmb = new Float32Array(emb.embedding.buffer, emb.embedding.byteOffset, emb.embedding.byteLength / 4);
+ const similarity = speakerEmbeddings.cosineSimilarity(profileEmb, speakerEmb);
+
+ if (similarity > 0.70) {
+ this.databaseManager.setSpeakerMapping(noteId, emb.speaker_id, profile.id, profile.display_name);
+
+ const note = this.databaseManager.getNote(noteId);
+ if (note?.transcript) {
+ try {
+ const segments = JSON.parse(note.transcript);
+ let changed = false;
+ for (const seg of segments) {
+ if (seg.speaker === emb.speaker_id && !seg.speakerName) {
+ seg.speakerName = profile.display_name;
+ changed = true;
+ }
+ }
+ if (changed) {
+ this.databaseManager.updateNote(noteId, { transcript: JSON.stringify(segments) });
+ }
+ } catch (_) {}
+ }
+ }
+ }
+ }
+ } catch (err) {
+ debugLogger.warn("Retroactive speaker mapping failed", { error: err.message });
+ }
+ });
}
_startOrSkipDiarization(sessionId, rawPcmPath, transcriptSegments, win) {
@@ -5124,7 +5215,82 @@ class IPCHandlers {
diarizationSegments
);
- send({ segments: enrichedSegments });
+ const speakerSet = new Set(diarizationSegments.map(d => d.speaker));
+ const speakerRenumber = new Map();
+ let sIdx = 0;
+ for (const sp of speakerSet) {
+ speakerRenumber.set(sp, `speaker_${sIdx}`);
+ sIdx++;
+ }
+
+ let speakerEmbeddingsMap = null;
+ const speakerEmb = require("./speakerEmbeddings");
+ try {
+ if (speakerEmb.isAvailable() && tmpWav) {
+ const speakerIds = [...new Set(diarizationSegments.map(s => s.speaker))];
+ speakerEmbeddingsMap = {};
+
+ for (const spk of speakerIds) {
+ const segs = diarizationSegments.filter(s => s.speaker === spk);
+ const sorted = segs.sort((a, b) => (b.end - b.start) - (a.end - a.start)).slice(0, 3);
+ const embeddings = [];
+ for (const seg of sorted) {
+ if (seg.end - seg.start < 2) continue;
+ const emb = await speakerEmb.extractEmbedding(tmpWav, seg.start, seg.end);
+ if (emb) embeddings.push(emb);
+ }
+ if (embeddings.length > 0) {
+ const centroid = speakerEmb.computeCentroid(embeddings);
+ const mappedId = speakerRenumber.get(spk) || spk;
+ speakerEmbeddingsMap[mappedId] = Array.from(centroid);
+ }
+ }
+ }
+ } catch (err) {
+ debugLogger.debug("Speaker embedding extraction skipped", { error: err.message });
+ }
+
+ if (speakerEmbeddingsMap) {
+ try {
+ const profiles = this.databaseManager.getSpeakerProfiles(true);
+
+ if (profiles.length > 0) {
+ for (const [mappedId, embArr] of Object.entries(speakerEmbeddingsMap)) {
+ const emb = new Float32Array(embArr);
+ let bestProfile = null;
+ let bestSim = 0;
+
+ for (const profile of profiles) {
+ const profileEmb = new Float32Array(profile.embedding.buffer, profile.embedding.byteOffset, profile.embedding.byteLength / 4);
+ const sim = speakerEmb.cosineSimilarity(emb, profileEmb);
+ if (sim > bestSim) {
+ bestSim = sim;
+ bestProfile = profile;
+ }
+ }
+
+ if (bestProfile && bestSim > 0.70) {
+ for (const seg of enrichedSegments) {
+ if (seg.speaker === mappedId) {
+ seg.speakerName = bestProfile.display_name;
+ }
+ }
+ } else if (bestProfile && bestSim > 0.55) {
+ for (const seg of enrichedSegments) {
+ if (seg.speaker === mappedId) {
+ seg.suggestedName = bestProfile.display_name;
+ seg.suggestedProfileId = bestProfile.id;
+ }
+ }
+ }
+ }
+ }
+ } catch (err) {
+ debugLogger.debug("Auto speaker recognition skipped", { error: err.message });
+ }
+ }
+
+ send({ segments: enrichedSegments, speakerEmbeddings: speakerEmbeddingsMap });
} catch (err) {
debugLogger.warn("Background diarization failed", { error: err.message });
send({ segments: [] });
diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.js
new file mode 100644
index 000000000..bcc2d4856
--- /dev/null
+++ b/src/helpers/speakerEmbeddings.js
@@ -0,0 +1,123 @@
+const fs = require("fs");
+const path = require("path");
+const debugLogger = require("./debugLogger");
+const { getModelsDirForService } = require("./modelDirUtils");
+
+const EMBEDDING_DIM = 512;
+const MIN_SEGMENT_SECONDS = 2;
+const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
+
+class SpeakerEmbeddings {
+ constructor() {
+ this.session = null;
+ this.modelPath = path.join(getModelsDirForService("diarization"), MODEL_FILE);
+ }
+
+ isAvailable() {
+ return fs.existsSync(this.modelPath);
+ }
+
+ async _ensureLoaded() {
+ if (this.session) return;
+
+ if (!this.isAvailable()) {
+ throw new Error(`Speaker embedding model not found at ${this.modelPath}`);
+ }
+
+ debugLogger.debug("speaker-embeddings loading model", { modelPath: this.modelPath });
+
+ const ort = require("onnxruntime-node");
+ this.session = await ort.InferenceSession.create(this.modelPath);
+
+ debugLogger.debug("speaker-embeddings model loaded");
+ }
+
+ async extractEmbedding(wavPath, startSec, endSec) {
+ if (endSec - startSec < MIN_SEGMENT_SECONDS) return null;
+
+ await this._ensureLoaded();
+
+ const buf = fs.readFileSync(wavPath);
+ const { sampleRate, dataOffset } = this._parseWavHeader(buf);
+
+ const startSample = Math.floor(startSec * sampleRate);
+ const endSample = Math.floor(endSec * sampleRate);
+ const numSamples = endSample - startSample;
+
+ const samples = new Float32Array(numSamples);
+ const bytesPerSample = 2;
+ const offset = dataOffset + startSample * bytesPerSample;
+
+ for (let i = 0; i < numSamples; i++) {
+ const bytePos = offset + i * bytesPerSample;
+ if (bytePos + 1 >= buf.length) break;
+ const int16 = buf.readInt16LE(bytePos);
+ samples[i] = int16 / 32768;
+ }
+
+ const ort = require("onnxruntime-node");
+ const inputName = this.session.inputNames[0];
+ const feeds = {
+ [inputName]: new ort.Tensor("float32", samples, [1, numSamples]),
+ };
+
+ const results = await this.session.run(feeds);
+ const output = results[Object.keys(results)[0]];
+
+ return new Float32Array(output.data);
+ }
+
+ _parseWavHeader(buf) {
+ let offset = 12;
+ let sampleRate = 16000;
+ let dataOffset = 44;
+
+ while (offset < buf.length - 8) {
+ const chunkId = buf.toString("ascii", offset, offset + 4);
+ const chunkSize = buf.readUInt32LE(offset + 4);
+
+ if (chunkId === "fmt ") {
+ sampleRate = buf.readUInt32LE(offset + 12);
+ } else if (chunkId === "data") {
+ dataOffset = offset + 8;
+ break;
+ }
+
+ offset += 8 + chunkSize;
+ }
+
+ return { sampleRate, dataOffset };
+ }
+
+ computeCentroid(embeddings) {
+ if (embeddings.length === 0) return new Float32Array(EMBEDDING_DIM);
+
+ const centroid = new Float32Array(EMBEDDING_DIM);
+ for (const emb of embeddings) {
+ for (let i = 0; i < EMBEDDING_DIM; i++) {
+ centroid[i] += emb[i];
+ }
+ }
+ for (let i = 0; i < EMBEDDING_DIM; i++) {
+ centroid[i] /= embeddings.length;
+ }
+ return centroid;
+ }
+
+ cosineSimilarity(a, b) {
+ let dot = 0;
+ let normA = 0;
+ let normB = 0;
+ for (let i = 0; i < a.length; i++) {
+ dot += a[i] * b[i];
+ normA += a[i] * a[i];
+ normB += b[i] * b[i];
+ }
+ const denom = Math.sqrt(normA) * Math.sqrt(normB);
+ return denom === 0 ? 0 : dot / denom;
+ }
+}
+
+const instance = new SpeakerEmbeddings();
+module.exports = instance;
+module.exports.SpeakerEmbeddings = SpeakerEmbeddings;
diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.ts
index 5a9379287..ca59814b1 100644
--- a/src/hooks/useMeetingTranscription.ts
+++ b/src/hooks/useMeetingTranscription.ts
@@ -9,6 +9,9 @@ export interface TranscriptSegment {
source: "mic" | "system";
timestamp?: number;
speaker?: string;
+ speakerName?: string;
+ suggestedName?: string;
+ suggestedProfileId?: number;
}
interface UseMeetingTranscriptionReturn {
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index 4ae978bb9..01629e8c9 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -1755,7 +1755,14 @@
"them": "Gegenüber",
"label": "Sprecher {{n}}",
"identifyingSpeakers": "Sprecher werden identifiziert...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "Namen eingeben",
+ "knownSpeakers": "Bekannte Sprecher",
+ "meetingAttendees": "Meeting-Teilnehmer",
+ "assignName": "Name zuweisen",
+ "isThisPerson": "Ist das {{name}}?",
+ "confirmed": "Bestätigt",
+ "nameAssigned": "Name zugewiesen"
}
},
"dictionary": {
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index 49b274007..c2dcb143f 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -1864,7 +1864,14 @@
"them": "Them",
"label": "Speaker {{n}}",
"identifyingSpeakers": "Identifying speakers...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "Type a name",
+ "knownSpeakers": "Known speakers",
+ "meetingAttendees": "Meeting attendees",
+ "assignName": "Assign name",
+ "isThisPerson": "Is this {{name}}?",
+ "confirmed": "Confirmed",
+ "nameAssigned": "Name assigned"
}
},
"dictionary": {
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 6a978fac4..5df95abf3 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -1755,7 +1755,14 @@
"them": "Ellos",
"label": "Participante {{n}}",
"identifyingSpeakers": "Identificando participantes...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "Escribe un nombre",
+ "knownSpeakers": "Hablantes conocidos",
+ "meetingAttendees": "Asistentes de la reunión",
+ "assignName": "Asignar nombre",
+ "isThisPerson": "¿Es {{name}}?",
+ "confirmed": "Confirmado",
+ "nameAssigned": "Nombre asignado"
}
},
"dictionary": {
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index e9b55686e..1a2f60d9d 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -1816,7 +1816,14 @@
"them": "Eux",
"label": "Intervenant {{n}}",
"identifyingSpeakers": "Identification des intervenants...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "Saisir un nom",
+ "knownSpeakers": "Interlocuteurs connus",
+ "meetingAttendees": "Participants à la réunion",
+ "assignName": "Attribuer un nom",
+ "isThisPerson": "Est-ce {{name}} ?",
+ "confirmed": "Confirmé",
+ "nameAssigned": "Nom attribué"
}
},
"dictionary": {
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index e18bb7c56..4dc5071b3 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -1755,7 +1755,14 @@
"them": "Loro",
"label": "Partecipante {{n}}",
"identifyingSpeakers": "Identificazione dei partecipanti...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "Digita un nome",
+ "knownSpeakers": "Parlanti noti",
+ "meetingAttendees": "Partecipanti alla riunione",
+ "assignName": "Assegna nome",
+ "isThisPerson": "È {{name}}?",
+ "confirmed": "Confermato",
+ "nameAssigned": "Nome assegnato"
}
},
"dictionary": {
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index c55e23da3..b8fd9be88 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -1683,7 +1683,14 @@
"them": "相手",
"label": "話者 {{n}}",
"identifyingSpeakers": "話者を識別中...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "名前を入力",
+ "knownSpeakers": "登録済みの話者",
+ "meetingAttendees": "会議の参加者",
+ "assignName": "名前を割り当て",
+ "isThisPerson": "{{name}}さんですか?",
+ "confirmed": "確認済み",
+ "nameAssigned": "名前を割り当てました"
}
},
"dictionary": {
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index 6e230841b..ed60065de 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -1755,7 +1755,14 @@
"them": "Outro",
"label": "Participante {{n}}",
"identifyingSpeakers": "Identificando participantes...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "Digite um nome",
+ "knownSpeakers": "Falantes conhecidos",
+ "meetingAttendees": "Participantes da reunião",
+ "assignName": "Atribuir nome",
+ "isThisPerson": "É {{name}}?",
+ "confirmed": "Confirmado",
+ "nameAssigned": "Nome atribuído"
}
},
"dictionary": {
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index 5bd82d74c..a61edeb37 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -1755,7 +1755,14 @@
"them": "Собеседник",
"label": "Участник {{n}}",
"identifyingSpeakers": "Определение участников...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "Введите имя",
+ "knownSpeakers": "Известные собеседники",
+ "meetingAttendees": "Участники встречи",
+ "assignName": "Назначить имя",
+ "isThisPerson": "Это {{name}}?",
+ "confirmed": "Подтверждено",
+ "nameAssigned": "Имя назначено"
}
},
"dictionary": {
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index 3201baec9..bbf29bddd 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -1755,7 +1755,14 @@
"them": "对方",
"label": "发言者 {{n}}",
"identifyingSpeakers": "正在识别发言者...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "输入姓名",
+ "knownSpeakers": "已知说话者",
+ "meetingAttendees": "会议参与者",
+ "assignName": "分配姓名",
+ "isThisPerson": "这是{{name}}吗?",
+ "confirmed": "已确认",
+ "nameAssigned": "姓名已分配"
}
},
"dictionary": {
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index 0bd9feb21..4d61a971c 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -1755,7 +1755,14 @@
"them": "對方",
"label": "發言者 {{n}}",
"identifyingSpeakers": "正在辨識發言者...",
- "speakersIdentified": "Speakers identified"
+ "speakersIdentified": "Speakers identified",
+ "typeNamePlaceholder": "輸入姓名",
+ "knownSpeakers": "已知發言者",
+ "meetingAttendees": "會議參與者",
+ "assignName": "指派姓名",
+ "isThisPerson": "這是{{name}}嗎?",
+ "confirmed": "已確認",
+ "nameAssigned": "姓名已指派"
}
},
"dictionary": {
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 15f8b9c49..29b246e67 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -1330,10 +1330,42 @@ declare global {
source: "mic" | "system";
timestamp?: number;
speaker?: string;
+ speakerName?: string;
+ suggestedName?: string;
+ suggestedProfileId?: number;
}>;
+ speakerEmbeddings?: Record<string, number[]> | null;
}) => void
) => () => void;
+ // Speaker name mapping
+ getSpeakerMappings?: (noteId: number) => Promise<Array<{
+ note_id: number;
+ speaker_id: string;
+ profile_id: number | null;
+ display_name: string;
+ }>>;
+ setSpeakerMapping?: (
+ noteId: number,
+ speakerId: string,
+ displayName: string,
+ email?: string | null,
+ profileId?: number | null
+ ) => Promise<{ success: boolean; profileId: number | null }>;
+ removeSpeakerMapping?: (noteId: number, speakerId: string) => Promise<{ success: boolean }>;
+ getSpeakerProfiles?: () => Promise<Array<{
+ id: number;
+ display_name: string;
+ email: string | null;
+ sample_count: number;
+ created_at: string;
+ updated_at: string;
+ }>>;
+ saveNoteSpeakerEmbeddings?: (
+ noteId: number,
+ embeddings: Record<string, number[]>
+ ) => Promise<{ success: boolean }>;
+
// Dictation realtime streaming
dictationRealtimeWarmup?: (options: {
model?: string;
diff --git a/src/utils/parseTranscriptSegments.ts b/src/utils/parseTranscriptSegments.ts
index e9ffce1ff..60e97bd46 100644
--- a/src/utils/parseTranscriptSegments.ts
+++ b/src/utils/parseTranscriptSegments.ts
@@ -8,6 +8,9 @@ export function parseTranscriptSegments(raw: string): TranscriptSegment[] {
source: "mic" | "system";
timestamp?: number;
speaker?: string;
+ speakerName?: string;
+ suggestedName?: string;
+ suggestedProfileId?: number;
}>;
return parsed.map((s, i) => ({
id: `stored-${i}`,
@@ -15,6 +18,9 @@ export function parseTranscriptSegments(raw: string): TranscriptSegment[] {
source: s.source,
timestamp: s.timestamp,
speaker: s.speaker,
+ speakerName: s.speakerName,
+ suggestedName: s.suggestedName,
+ suggestedProfileId: s.suggestedProfileId,
}));
} catch (e) {
logger.warn("Failed to parse transcript segments", e);