Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat(notes): add speaker recognition for meeting transcripts
Gabriel Stein616576fb906c
diff --git a/preload.js b/preload.jsindex 8f48dcce4..a297c95cd 100644--- a/preload.js+++ b/preload.js@@ -263,6 +263,16 @@ contextBridge.exposeInMainWorld("electronAPI", {(callback) => (_event, data) => callback(data)),+ // Speaker name mapping+ getSpeakerMappings: (noteId) => ipcRenderer.invoke("get-speaker-mappings", noteId),+ setSpeakerMapping: (noteId, speakerId, displayName, email, profileId) =>+ ipcRenderer.invoke("set-speaker-mapping", noteId, speakerId, displayName, email, profileId),+ removeSpeakerMapping: (noteId, speakerId) =>+ ipcRenderer.invoke("remove-speaker-mapping", noteId, speakerId),+ getSpeakerProfiles: () => ipcRenderer.invoke("get-speaker-profiles"),+ saveNoteSpeakerEmbeddings: (noteId, embeddings) =>+ ipcRenderer.invoke("save-note-speaker-embeddings", noteId, embeddings),+// Window control functionswindowMinimize: () => ipcRenderer.invoke("window-minimize"),windowMaximize: () => ipcRenderer.invoke("window-maximize"),diff --git a/src/components/notes/MeetingTranscriptChat.tsx b/src/components/notes/MeetingTranscriptChat.tsxindex 5ee6a292c..510517593 100644--- a/src/components/notes/MeetingTranscriptChat.tsx+++ b/src/components/notes/MeetingTranscriptChat.tsx@@ -1,5 +1,7 @@-import { useEffect, useRef } from "react";+import { useState, useEffect, useRef } from "react";import { useTranslation } from "react-i18next";+import { Check, X } from "lucide-react";+import { Popover, PopoverTrigger, PopoverContent } from "../ui/popover";import { cn } from "../lib/utils";import type { TranscriptSegment } from "../../hooks/useMeetingTranscription";@@ -72,16 +74,209 @@ function PartialBubble({ text, source }: { text: string; source: "mic" | "system);}+function SpeakerLabel({+ speakerId,+ segment,+ mappedName,+ speakerProfiles,+ participants,+ colorIdx,+ isYou,+ onMap,+ onConfirm,+ onDismiss,+ t,+}: {+ speakerId: string;+ segment: TranscriptSegment;+ mappedName?: string;+ speakerProfiles?: Array<{ id: number; display_name: string; email: string | null }>;+ participants?: Array<{ email: string; displayName: string | null }>;+ colorIdx: number;+ isYou: boolean;+ onMap?: (speakerId: string, name: string, email?: string | null, profileId?: number) => void;+ onConfirm?: (speakerId: string, name: string, profileId: number) => void;+ onDismiss?: (speakerId: string) => void;+ t: (key: string, opts?: Record<string, unknown>) => string;+}) {+ const [open, setOpen] = useState(false);+ const [search, setSearch] = useState("");++ if (isYou) {+ return (+ <span className="text-[11px] font-medium mb-0.5 px-1 text-primary/60">+ {t("notes.speaker.you")}+ </span>+ );+ }++ const hasSuggestion = !!segment.suggestedName && !mappedName;++ if (hasSuggestion) {+ return (+ <span className="group inline-flex items-center gap-1 mb-0.5 px-1">+ <span className="text-[11px] font-medium italic text-muted-foreground/60">+ {segment.suggestedName}+ </span>+ <button+ onClick={() =>+ onConfirm?.(speakerId, segment.suggestedName!, segment.suggestedProfileId!)+ }+ className="opacity-0 group-hover:opacity-100 p-0.5 rounded transition-opacity cursor-pointer text-muted-foreground hover:text-emerald-500"+ >+ <Check size={12} />+ </button>+ <button+ onClick={() => onDismiss?.(speakerId)}+ className="opacity-0 group-hover:opacity-100 p-0.5 rounded transition-opacity cursor-pointer text-muted-foreground hover:text-destructive"+ >+ <X size={12} />+ </button>+ </span>+ );+ }++ const speakerNumber = (() => {+ const match = speakerId.match(/speaker_(\d+)/);+ return match ? Number(match[1]) + 1 : 1;+ })();++ const displayLabel = mappedName || segment.speakerName || t("notes.speaker.label", { n: speakerNumber });+ const isUnmapped = !mappedName && !segment.speakerName;++ const filteredParticipants = participants?.filter(+ (p) =>+ search === "" ||+ (p.displayName || p.email).toLowerCase().includes(search.toLowerCase())+ );++ const filteredProfiles = speakerProfiles?.filter(+ (p) =>+ search === "" ||+ p.display_name.toLowerCase().includes(search.toLowerCase()) ||+ (p.email && p.email.toLowerCase().includes(search.toLowerCase()))+ );++ const handleSelect = (name: string, email?: string | null, profileId?: number) => {+ onMap?.(speakerId, name, email, profileId);+ setOpen(false);+ setSearch("");+ };++ const handleKeyDown = (e: React.KeyboardEvent) => {+ if (e.key === "Enter" && search.trim()) {+ e.preventDefault();+ handleSelect(search.trim());+ }+ };++ return (+ <Popover+ open={open}+ onOpenChange={(o) => {+ setOpen(o);+ if (!o) setSearch("");+ }}+ >+ <PopoverTrigger asChild>+ <button+ className={cn(+ "text-[11px] font-medium mb-0.5 px-1 transition-colors duration-100 cursor-pointer rounded-sm outline-none focus-visible:ring-1 focus-visible:ring-ring",+ SPEAKER_COLORS[colorIdx],+ isUnmapped && "border-b border-dotted border-current",+ "hover:bg-muted/50"+ )}+ >+ {displayLabel}+ </button>+ </PopoverTrigger>+ <PopoverContent className="w-72 p-0">+ <div className="p-2 border-b border-border/50">+ <input+ value={search}+ onChange={(e) => setSearch(e.target.value)}+ onKeyDown={handleKeyDown}+ placeholder={t("notes.speaker.typeNamePlaceholder")}+ className="w-full px-2 py-1.5 rounded-md bg-transparent text-xs text-foreground placeholder:text-foreground/20 outline-none border-none appearance-none"+ autoFocus+ />+ </div>+ <div className="max-h-40 overflow-y-auto">+ {filteredParticipants && filteredParticipants.length > 0 && (+ <div className="p-1 border-b border-border/30">+ <div className="px-2 py-1 text-[11px] font-medium text-muted-foreground">+ {t("notes.speaker.meetingAttendees")}+ </div>+ {filteredParticipants.slice(0, 5).map((p) => (+ <button+ key={p.email}+ onClick={() => handleSelect(p.displayName || p.email.split("@")[0], p.email)}+ className="flex items-center gap-2 w-full px-2 py-1.5 rounded-md text-xs text-foreground/70 hover:bg-foreground/5 transition-colors cursor-pointer"+ >+ <span className="truncate">{p.displayName || p.email}</span>+ </button>+ ))}+ </div>+ )}+ {filteredProfiles && filteredProfiles.length > 0 && (+ <div className="p-1">+ <div className="px-2 py-1 text-[11px] font-medium text-muted-foreground">+ {t("notes.speaker.knownSpeakers")}+ </div>+ {filteredProfiles.slice(0, 5).map((p) => (+ <button+ key={p.id}+ onClick={() => handleSelect(p.display_name, p.email, p.id)}+ className="flex items-center gap-2 w-full px-2 py-1.5 rounded-md text-xs text-foreground/70 hover:bg-foreground/5 transition-colors cursor-pointer"+ >+ <span className="truncate">{p.display_name}</span>+ {p.email && (+ <span className="text-foreground/30 truncate text-[11px]">{p.email}</span>+ )}+ </button>+ ))}+ </div>+ )}+ {(!filteredParticipants || filteredParticipants.length === 0) &&+ (!filteredProfiles || filteredProfiles.length === 0) &&+ !search && (+ <div className="px-3 py-4 text-center text-[11px] text-foreground/30">+ {t("notes.speaker.typeNamePlaceholder")}+ </div>+ )}+ </div>+ </PopoverContent>+ </Popover>+ );+}+interface MeetingTranscriptChatProps {segments: TranscriptSegment[];micPartial?: string;systemPartial?: string;+ speakerMappings?: Record<string, string>;+ speakerProfiles?: Array<{ id: number; display_name: string; email: string | null }>;+ participants?: Array<{ email: string; displayName: string | null }>;+ onMapSpeaker?: (+ speakerId: string,+ displayName: string,+ email?: string | null,+ profileId?: number+ ) => void;+ onConfirmSuggestion?: (speakerId: string, suggestedName: string, profileId: number) => void;+ onDismissSuggestion?: (speakerId: string) => void;}export function MeetingTranscriptChat({segments,micPartial,systemPartial,+ speakerMappings,+ speakerProfiles,+ participants,+ onMapSpeaker,+ onConfirmSuggestion,+ onDismissSuggestion,}: MeetingTranscriptChatProps) {const { t } = useTranslation();const scrollRef = useRef<HTMLDivElement>(null);@@ -89,7 +284,6 @@ export function MeetingTranscriptChat({useEffect(() => {const el = scrollRef.current;if (!el) return;- // Auto-scroll only when near the bottom (within 80px)const isNearBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 80;if (isNearBottom) {el.scrollTop = el.scrollHeight;@@ -125,17 +319,6 @@ export function MeetingTranscriptChat({const isSystemSpeaker = hasSpeaker && !isYou;const colorIdx = isSystemSpeaker ? getSpeakerColorIndex(segment.speaker!) : 0;- let speakerLabel: string | null = null;- if (hasSpeaker && !sameSpeaker) {- if (isYou) {- speakerLabel = t("notes.speaker.you");- } else {- const match = segment.speaker!.match(/speaker_(\d+)/);- const n = match ? Number(match[1]) + 1 : 1;- speakerLabel = t("notes.speaker.label", { n });- }- }-return (<divkey={segment.id}@@ -146,15 +329,20 @@ export function MeetingTranscriptChat({)}style={{ animation: "agent-message-in 200ms ease-out both" }}>- {speakerLabel && (- <span- className={cn(- "text-[11px] font-medium mb-0.5 px-1",- isYou ? "text-primary/60" : SPEAKER_COLORS[colorIdx]- )}- >- {speakerLabel}- </span>+ {hasSpeaker && !sameSpeaker && (+ <SpeakerLabel+ speakerId={segment.speaker!}+ segment={segment}+ mappedName={speakerMappings?.[segment.speaker!]}+ speakerProfiles={speakerProfiles}+ participants={participants}+ colorIdx={colorIdx}+ isYou={isYou}+ onMap={onMapSpeaker}+ onConfirm={onConfirmSuggestion}+ onDismiss={onDismissSuggestion}+ t={t}+ />)}<divclassName={cn(diff --git a/src/components/notes/NoteEditor.tsx b/src/components/notes/NoteEditor.tsxindex 37e591426..60ee6ef9b 100644--- a/src/components/notes/NoteEditor.tsx+++ b/src/components/notes/NoteEditor.tsx@@ -37,6 +37,20 @@ import { parseTranscriptSegments } from "../../utils/parseTranscriptSegments";import NoteParticipants from "./NoteParticipants";import type { CalendarAttendee } from "../../types/calendar";+function serializeTranscript(segments: TranscriptSegment[]): string {+ return JSON.stringify(+ segments.map((s) => ({+ text: s.text,+ source: s.source,+ timestamp: s.timestamp,+ speaker: s.speaker,+ speakerName: s.speakerName,+ suggestedName: s.suggestedName,+ suggestedProfileId: s.suggestedProfileId,+ }))+ );+}+function formatNoteDate(dateStr: string): string {const date = normalizeDbDate(dateStr);if (Number.isNaN(date.getTime())) return "";@@ -128,6 +142,10 @@ export default function NoteEditor({const [newFolderName, setNewFolderName] = useState("");const [isDiarizing, setIsDiarizing] = useState(false);const [diarizedSegments, setDiarizedSegments] = useState<TranscriptSegment[] | null>(null);+ const [speakerMappings, setSpeakerMappings] = useState<Record<string, string>>({});+ const [speakerProfiles, setSpeakerProfiles] = useState<+ Array<{ id: number; display_name: string; email: string | null }>+ >([]);const editorRef = useRef<Editor | null>(null);const embeddedChat = useEmbeddedChat({@@ -214,6 +232,7 @@ export default function NoteEditor({setChatMode("hidden");setDiarizedSegments(null);setIsDiarizing(false);+ setSpeakerMappings({});if (!isMeetingRecording) {setViewMode("raw");}@@ -224,6 +243,19 @@ export default function NoteEditor({}}, [note.id, isMeetingRecording]);+ useEffect(() => {+ window.electronAPI?.getSpeakerMappings?.(note.id).then((mappings) => {+ const map: Record<string, string> = {};+ for (const m of mappings || []) map[m.speaker_id] = m.display_name;+ setSpeakerMappings(map);+ });+ window.electronAPI?.getSpeakerProfiles?.().then((profiles) => {+ setSpeakerProfiles(+ (profiles || []).map((p) => ({ id: p.id, display_name: p.display_name, email: p.email }))+ );+ });+ }, [note.id]);+// Auto-show chat when hook loads an existing conversation with messagesuseEffect(() => {if (@@ -265,19 +297,86 @@ export default function NoteEditor({}));setDiarizedSegments(enriched);- const serialized = JSON.stringify(- enriched.map((s: TranscriptSegment) => ({- text: s.text,- source: s.source,- timestamp: s.timestamp,- speaker: s.speaker,- }))- );- window.electronAPI.updateNote(note.id, { transcript: serialized });+ window.electronAPI.updateNote(note.id, { transcript: serializeTranscript(enriched) });++ if (data.speakerEmbeddings) {+ window.electronAPI?.saveNoteSpeakerEmbeddings?.(note.id, data.speakerEmbeddings);+ }++ const autoMappings: Record<string, string> = {};+ for (const s of enriched) {+ if (s.speakerName && s.speaker) autoMappings[s.speaker] = s.speakerName;+ }+ if (Object.keys(autoMappings).length > 0) {+ setSpeakerMappings((prev) => ({ ...prev, ...autoMappings }));+ }});return () => cleanup?.();}, [note.id, diarizationSessionId]);+ const handleMapSpeaker = useCallback(+ async (+ speakerId: string,+ displayName: string,+ email?: string | null,+ profileId?: number | null+ ) => {+ setSpeakerMappings((prev) => ({ ...prev, [speakerId]: displayName }));+ await window.electronAPI?.setSpeakerMapping?.(+ note.id,+ speakerId,+ displayName,+ email,+ profileId+ );++ const currentSegments = displaySegments.map((s) =>+ s.speaker === speakerId+ ? {+ ...s,+ speakerName: displayName,+ suggestedName: undefined,+ suggestedProfileId: undefined,+ }+ : s+ );+ window.electronAPI?.updateNote(note.id, { transcript: serializeTranscript(currentSegments) });++ window.electronAPI?.getSpeakerProfiles?.().then((profiles) => {+ setSpeakerProfiles(+ (profiles || []).map((p) => ({+ id: p.id,+ display_name: p.display_name,+ email: p.email,+ }))+ );+ });+ },+ [note.id, displaySegments]+ );++ const handleConfirmSuggestion = useCallback(+ async (speakerId: string, suggestedName: string, profileId: number) => {+ await handleMapSpeaker(speakerId, suggestedName, null, profileId);+ },+ [handleMapSpeaker]+ );++ const handleDismissSuggestion = useCallback(+ async (speakerId: string) => {+ const currentSegments = displaySegments.map((s) =>+ s.speaker === speakerId+ ? { ...s, suggestedName: undefined, suggestedProfileId: undefined }+ : s+ );+ if (diarizedSegments) {+ setDiarizedSegments(currentSegments);+ }+ window.electronAPI?.updateNote(note.id, { transcript: serializeTranscript(currentSegments) });+ },+ [note.id, displaySegments, diarizedSegments]+ );+const handleTitleInput = useCallback(() => {if (titleRef.current) {const text = titleRef.current.textContent || "";@@ -587,6 +686,12 @@ export default function NoteEditor({segments={displaySegments}micPartial={isMeetingRecording ? meetingMicPartial : undefined}systemPartial={isMeetingRecording ? meetingSystemPartial : undefined}+ speakerMappings={speakerMappings}+ speakerProfiles={speakerProfiles}+ participants={parsedParticipants}+ onMapSpeaker={handleMapSpeaker}+ onConfirmSuggestion={handleConfirmSuggestion}+ onDismissSuggestion={handleDismissSuggestion}/>{isDiarizing && (<div className="flex items-center justify-center gap-1.5 py-2">diff --git a/src/helpers/database.js b/src/helpers/database.jsindex 52f407528..63fa903e0 100644--- a/src/helpers/database.js+++ b/src/helpers/database.js@@ -383,6 +383,40 @@ class DatabaseManager {)`);+ this.db.exec(`+ CREATE TABLE IF NOT EXISTS speaker_profiles (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ display_name TEXT NOT NULL,+ email TEXT,+ embedding BLOB NOT NULL,+ sample_count INTEGER DEFAULT 1,+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP,+ updated_at DATETIME DEFAULT CURRENT_TIMESTAMP+ )+ `);++ this.db.exec(`+ CREATE TABLE IF NOT EXISTS speaker_mappings (+ note_id INTEGER NOT NULL,+ speaker_id TEXT NOT NULL,+ profile_id INTEGER,+ display_name TEXT NOT NULL,+ PRIMARY KEY (note_id, speaker_id),+ FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE CASCADE,+ FOREIGN KEY (profile_id) REFERENCES speaker_profiles(id) ON DELETE SET NULL+ )+ `);++ this.db.exec(`+ CREATE TABLE IF NOT EXISTS note_speaker_embeddings (+ note_id INTEGER NOT NULL,+ speaker_id TEXT NOT NULL,+ embedding BLOB NOT NULL,+ PRIMARY KEY (note_id, speaker_id),+ FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE CASCADE+ )+ `);+return true;} catch (error) {debugLogger.error("Database initialization failed", { error: error.message }, "database");@@ -1482,6 +1516,164 @@ class DatabaseManager {throw error;}}++ upsertSpeakerProfile(name, email, embeddingBuffer, profileId = null) {+ try {+ if (!this.db) throw new Error("Database not initialized");+ const existing = profileId+ ? this.db.prepare("SELECT * FROM speaker_profiles WHERE id = ?").get(profileId)+ : this.db.prepare("SELECT * FROM speaker_profiles WHERE display_name = ?").get(name);+ if (existing) {+ const stored = new Float32Array(+ existing.embedding.buffer,+ existing.embedding.byteOffset,+ existing.embedding.byteLength / 4+ );+ const incoming = new Float32Array(+ embeddingBuffer.buffer,+ embeddingBuffer.byteOffset,+ embeddingBuffer.byteLength / 4+ );+ const updated = new Float32Array(stored.length);+ for (let i = 0; i < stored.length; i++) {+ updated[i] = 0.3 * incoming[i] + 0.7 * stored[i];+ }+ const updatedBuf = Buffer.from(updated.buffer);+ this.db+ .prepare(+ "UPDATE speaker_profiles SET display_name = ?, email = ?, embedding = ?, sample_count = sample_count + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?"+ )+ .run(name, email || existing.email || null, updatedBuf, existing.id);+ return this.db.prepare("SELECT * FROM speaker_profiles WHERE id = ?").get(existing.id);+ }+ const result = this.db+ .prepare(+ "INSERT INTO speaker_profiles (display_name, email, embedding) VALUES (?, ?, ?)"+ )+ .run(name, email || null, embeddingBuffer);+ return this.db+ .prepare("SELECT * FROM speaker_profiles WHERE id = ?")+ .get(result.lastInsertRowid);+ } catch (error) {+ debugLogger.error("Error upserting speaker profile", { error: error.message }, "database");+ throw error;+ }+ }++ getSpeakerProfiles(includeEmbedding = false) {+ try {+ if (!this.db) throw new Error("Database not initialized");+ const query = includeEmbedding+ ? "SELECT * FROM speaker_profiles"+ : `SELECT id, display_name, email, sample_count, created_at, updated_at+ FROM speaker_profiles`;+ return this.db.prepare(query).all();+ } catch (error) {+ debugLogger.error("Error getting speaker profiles", { error: error.message }, "database");+ throw error;+ }+ }++ setSpeakerMapping(noteId, speakerId, profileId, displayName) {+ try {+ if (!this.db) throw new Error("Database not initialized");+ this.db+ .prepare(+ "INSERT OR REPLACE INTO speaker_mappings (note_id, speaker_id, profile_id, display_name) VALUES (?, ?, ?, ?)"+ )+ .run(noteId, speakerId, profileId, displayName);+ return { success: true };+ } catch (error) {+ debugLogger.error("Error setting speaker mapping", { error: error.message }, "database");+ throw error;+ }+ }++ getSpeakerMappings(noteId) {+ try {+ if (!this.db) throw new Error("Database not initialized");+ return this.db+ .prepare("SELECT * FROM speaker_mappings WHERE note_id = ?")+ .all(noteId);+ } catch (error) {+ debugLogger.error("Error getting speaker mappings", { error: error.message }, "database");+ throw error;+ }+ }++ saveNoteSpeakerEmbeddings(noteId, embeddings) {+ try {+ if (!this.db) throw new Error("Database not initialized");+ const transaction = this.db.transaction((entries) => {+ const stmt = this.db.prepare(+ "INSERT OR REPLACE INTO note_speaker_embeddings (note_id, speaker_id, embedding) VALUES (?, ?, ?)"+ );+ for (const [speakerId, buffer] of entries) {+ stmt.run(noteId, speakerId, buffer);+ }+ });+ transaction(Object.entries(embeddings));+ return { success: true };+ } catch (error) {+ debugLogger.error(+ "Error saving note speaker embeddings",+ { error: error.message },+ "database"+ );+ throw error;+ }+ }++ getNoteSpeakerEmbeddings(noteId) {+ try {+ if (!this.db) throw new Error("Database not initialized");+ return this.db+ .prepare("SELECT * FROM note_speaker_embeddings WHERE note_id = ?")+ .all(noteId);+ } catch (error) {+ debugLogger.error(+ "Error getting note speaker embeddings",+ { error: error.message },+ "database"+ );+ throw error;+ }+ }++ getNotesWithUnmappedSpeakers() {+ try {+ if (!this.db) throw new Error("Database not initialized");+ return this.db+ .prepare(+ `SELECT DISTINCT nse.note_id+ FROM note_speaker_embeddings nse+ LEFT JOIN speaker_mappings sm ON nse.note_id = sm.note_id AND nse.speaker_id = sm.speaker_id+ WHERE sm.note_id IS NULL`+ )+ .all()+ .map((row) => row.note_id);+ } catch (error) {+ debugLogger.error(+ "Error getting notes with unmapped speakers",+ { error: error.message },+ "database"+ );+ throw error;+ }+ }++ removeSpeakerMapping(noteId, speakerId) {+ try {+ if (!this.db) throw new Error("Database not initialized");+ this.db+ .prepare("DELETE FROM speaker_mappings WHERE note_id = ? AND speaker_id = ?")+ .run(noteId, speakerId);+ return { success: true };+ } catch (error) {+ debugLogger.error("Error removing speaker mapping", { error: error.message }, "database");+ throw error;+ }+ }}module.exports = DatabaseManager;diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 66ef6a9b2..91453cccf 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -5084,6 +5084,97 @@ class IPCHandlers {return { canceled: true };}});++ ipcMain.handle("get-speaker-mappings", async (_event, noteId) => {+ return this.databaseManager.getSpeakerMappings(noteId);+ });++ ipcMain.handle(+ "set-speaker-mapping",+ async (_event, noteId, speakerId, displayName, email, profileId) => {+ const embeddings = this.databaseManager.getNoteSpeakerEmbeddings(noteId);+ const speakerEmb = embeddings.find((e) => e.speaker_id === speakerId);++ let resolvedProfileId = profileId ?? null;+ if (speakerEmb) {+ const profile = this.databaseManager.upsertSpeakerProfile(+ displayName,+ email || null,+ speakerEmb.embedding,+ resolvedProfileId+ );+ resolvedProfileId = profile.id;+ this._retroactiveMapping(profile);+ }++ this.databaseManager.setSpeakerMapping(noteId, speakerId, resolvedProfileId, displayName);+ return { success: true, profileId: resolvedProfileId };+ }+ );++ ipcMain.handle("remove-speaker-mapping", async (_event, noteId, speakerId) => {+ this.databaseManager.removeSpeakerMapping(noteId, speakerId);+ return { success: true };+ });++ ipcMain.handle("get-speaker-profiles", async () => {+ return this.databaseManager.getSpeakerProfiles();+ });++ ipcMain.handle("save-note-speaker-embeddings", async (_event, noteId, embeddingsObj) => {+ const buffers = {};+ for (const [speakerId, arr] of Object.entries(embeddingsObj)) {+ buffers[speakerId] = Buffer.from(new Float32Array(arr).buffer);+ }+ this.databaseManager.saveNoteSpeakerEmbeddings(noteId, buffers);+ return { success: true };+ });+ }++ _retroactiveMapping(profile) {+ setImmediate(async () => {+ try {+ const speakerEmbeddings = require("./speakerEmbeddings");+ const noteIds = this.databaseManager.getNotesWithUnmappedSpeakers();++ const profileEmb = new Float32Array(profile.embedding.buffer, profile.embedding.byteOffset, profile.embedding.byteLength / 4);++ for (const noteId of noteIds) {+ const embeddings = this.databaseManager.getNoteSpeakerEmbeddings(noteId);+ const existing = this.databaseManager.getSpeakerMappings(noteId);+ const mappedSpeakers = new Set(existing.map(m => m.speaker_id));+ for (const emb of embeddings) {+ if (mappedSpeakers.has(emb.speaker_id)) continue;++ const speakerEmb = new Float32Array(emb.embedding.buffer, emb.embedding.byteOffset, emb.embedding.byteLength / 4);+ const similarity = speakerEmbeddings.cosineSimilarity(profileEmb, speakerEmb);++ if (similarity > 0.70) {+ this.databaseManager.setSpeakerMapping(noteId, emb.speaker_id, profile.id, profile.display_name);++ const note = this.databaseManager.getNote(noteId);+ if (note?.transcript) {+ try {+ const segments = JSON.parse(note.transcript);+ let changed = false;+ for (const seg of segments) {+ if (seg.speaker === emb.speaker_id && !seg.speakerName) {+ seg.speakerName = profile.display_name;+ changed = true;+ }+ }+ if (changed) {+ this.databaseManager.updateNote(noteId, { transcript: JSON.stringify(segments) });+ }+ } catch (_) {}+ }+ }+ }+ }+ } catch (err) {+ debugLogger.warn("Retroactive speaker mapping failed", { error: err.message });+ }+ });}_startOrSkipDiarization(sessionId, rawPcmPath, transcriptSegments, win) {@@ -5124,7 +5215,82 @@ class IPCHandlers {diarizationSegments);- send({ segments: enrichedSegments });+ const speakerSet = new Set(diarizationSegments.map(d => d.speaker));+ const speakerRenumber = new Map();+ let sIdx = 0;+ for (const sp of speakerSet) {+ speakerRenumber.set(sp, `speaker_${sIdx}`);+ sIdx++;+ }++ let speakerEmbeddingsMap = null;+ const speakerEmb = require("./speakerEmbeddings");+ try {+ if (speakerEmb.isAvailable() && tmpWav) {+ const speakerIds = [...new Set(diarizationSegments.map(s => s.speaker))];+ speakerEmbeddingsMap = {};++ for (const spk of speakerIds) {+ const segs = diarizationSegments.filter(s => s.speaker === spk);+ const sorted = segs.sort((a, b) => (b.end - b.start) - (a.end - a.start)).slice(0, 3);+ const embeddings = [];+ for (const seg of sorted) {+ if (seg.end - seg.start < 2) continue;+ const emb = await speakerEmb.extractEmbedding(tmpWav, seg.start, seg.end);+ if (emb) embeddings.push(emb);+ }+ if (embeddings.length > 0) {+ const centroid = speakerEmb.computeCentroid(embeddings);+ const mappedId = speakerRenumber.get(spk) || spk;+ speakerEmbeddingsMap[mappedId] = Array.from(centroid);+ }+ }+ }+ } catch (err) {+ debugLogger.debug("Speaker embedding extraction skipped", { error: err.message });+ }++ if (speakerEmbeddingsMap) {+ try {+ const profiles = this.databaseManager.getSpeakerProfiles(true);++ if (profiles.length > 0) {+ for (const [mappedId, embArr] of Object.entries(speakerEmbeddingsMap)) {+ const emb = new Float32Array(embArr);+ let bestProfile = null;+ let bestSim = 0;++ for (const profile of profiles) {+ const profileEmb = new Float32Array(profile.embedding.buffer, profile.embedding.byteOffset, profile.embedding.byteLength / 4);+ const sim = speakerEmb.cosineSimilarity(emb, profileEmb);+ if (sim > bestSim) {+ bestSim = sim;+ bestProfile = profile;+ }+ }++ if (bestProfile && bestSim > 0.70) {+ for (const seg of enrichedSegments) {+ if (seg.speaker === mappedId) {+ seg.speakerName = bestProfile.display_name;+ }+ }+ } else if (bestProfile && bestSim > 0.55) {+ for (const seg of enrichedSegments) {+ if (seg.speaker === mappedId) {+ seg.suggestedName = bestProfile.display_name;+ seg.suggestedProfileId = bestProfile.id;+ }+ }+ }+ }+ }+ } catch (err) {+ debugLogger.debug("Auto speaker recognition skipped", { error: err.message });+ }+ }++ send({ segments: enrichedSegments, speakerEmbeddings: speakerEmbeddingsMap });} catch (err) {debugLogger.warn("Background diarization failed", { error: err.message });send({ segments: [] });diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.jsnew file mode 100644index 000000000..bcc2d4856--- /dev/null+++ b/src/helpers/speakerEmbeddings.js@@ -0,0 +1,123 @@+const fs = require("fs");+const path = require("path");+const debugLogger = require("./debugLogger");+const { getModelsDirForService } = require("./modelDirUtils");++const EMBEDDING_DIM = 512;+const MIN_SEGMENT_SECONDS = 2;+const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";++class SpeakerEmbeddings {+ constructor() {+ this.session = null;+ this.modelPath = path.join(getModelsDirForService("diarization"), MODEL_FILE);+ }++ isAvailable() {+ return fs.existsSync(this.modelPath);+ }++ async _ensureLoaded() {+ if (this.session) return;++ if (!this.isAvailable()) {+ throw new Error(`Speaker embedding model not found at ${this.modelPath}`);+ }++ debugLogger.debug("speaker-embeddings loading model", { modelPath: this.modelPath });++ const ort = require("onnxruntime-node");+ this.session = await ort.InferenceSession.create(this.modelPath);++ debugLogger.debug("speaker-embeddings model loaded");+ }++ async extractEmbedding(wavPath, startSec, endSec) {+ if (endSec - startSec < MIN_SEGMENT_SECONDS) return null;++ await this._ensureLoaded();++ const buf = fs.readFileSync(wavPath);+ const { sampleRate, dataOffset } = this._parseWavHeader(buf);++ const startSample = Math.floor(startSec * sampleRate);+ const endSample = Math.floor(endSec * sampleRate);+ const numSamples = endSample - startSample;++ const samples = new Float32Array(numSamples);+ const bytesPerSample = 2;+ const offset = dataOffset + startSample * bytesPerSample;++ for (let i = 0; i < numSamples; i++) {+ const bytePos = offset + i * bytesPerSample;+ if (bytePos + 1 >= buf.length) break;+ const int16 = buf.readInt16LE(bytePos);+ samples[i] = int16 / 32768;+ }++ const ort = require("onnxruntime-node");+ const inputName = this.session.inputNames[0];+ const feeds = {+ [inputName]: new ort.Tensor("float32", samples, [1, numSamples]),+ };++ const results = await this.session.run(feeds);+ const output = results[Object.keys(results)[0]];++ return new Float32Array(output.data);+ }++ _parseWavHeader(buf) {+ let offset = 12;+ let sampleRate = 16000;+ let dataOffset = 44;++ while (offset < buf.length - 8) {+ const chunkId = buf.toString("ascii", offset, offset + 4);+ const chunkSize = buf.readUInt32LE(offset + 4);++ if (chunkId === "fmt ") {+ sampleRate = buf.readUInt32LE(offset + 12);+ } else if (chunkId === "data") {+ dataOffset = offset + 8;+ break;+ }++ offset += 8 + chunkSize;+ }++ return { sampleRate, dataOffset };+ }++ computeCentroid(embeddings) {+ if (embeddings.length === 0) return new Float32Array(EMBEDDING_DIM);++ const centroid = new Float32Array(EMBEDDING_DIM);+ for (const emb of embeddings) {+ for (let i = 0; i < EMBEDDING_DIM; i++) {+ centroid[i] += emb[i];+ }+ }+ for (let i = 0; i < EMBEDDING_DIM; i++) {+ centroid[i] /= embeddings.length;+ }+ return centroid;+ }++ cosineSimilarity(a, b) {+ let dot = 0;+ let normA = 0;+ let normB = 0;+ for (let i = 0; i < a.length; i++) {+ dot += a[i] * b[i];+ normA += a[i] * a[i];+ normB += b[i] * b[i];+ }+ const denom = Math.sqrt(normA) * Math.sqrt(normB);+ return denom === 0 ? 0 : dot / denom;+ }+}++const instance = new SpeakerEmbeddings();+module.exports = instance;+module.exports.SpeakerEmbeddings = SpeakerEmbeddings;diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.tsindex 5a9379287..ca59814b1 100644--- a/src/hooks/useMeetingTranscription.ts+++ b/src/hooks/useMeetingTranscription.ts@@ -9,6 +9,9 @@ export interface TranscriptSegment {source: "mic" | "system";timestamp?: number;speaker?: string;+ speakerName?: string;+ suggestedName?: string;+ suggestedProfileId?: number;}interface UseMeetingTranscriptionReturn {diff --git a/src/locales/de/translation.json b/src/locales/de/translation.jsonindex 4ae978bb9..01629e8c9 100644--- a/src/locales/de/translation.json+++ b/src/locales/de/translation.json@@ -1755,7 +1755,14 @@"them": "Gegenüber","label": "Sprecher {{n}}","identifyingSpeakers": "Sprecher werden identifiziert...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "Namen eingeben",+ "knownSpeakers": "Bekannte Sprecher",+ "meetingAttendees": "Meeting-Teilnehmer",+ "assignName": "Name zuweisen",+ "isThisPerson": "Ist das {{name}}?",+ "confirmed": "Bestätigt",+ "nameAssigned": "Name zugewiesen"}},"dictionary": {diff --git a/src/locales/en/translation.json b/src/locales/en/translation.jsonindex 49b274007..c2dcb143f 100644--- a/src/locales/en/translation.json+++ b/src/locales/en/translation.json@@ -1864,7 +1864,14 @@"them": "Them","label": "Speaker {{n}}","identifyingSpeakers": "Identifying speakers...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "Type a name",+ "knownSpeakers": "Known speakers",+ "meetingAttendees": "Meeting attendees",+ "assignName": "Assign name",+ "isThisPerson": "Is this {{name}}?",+ "confirmed": "Confirmed",+ "nameAssigned": "Name assigned"}},"dictionary": {diff --git a/src/locales/es/translation.json b/src/locales/es/translation.jsonindex 6a978fac4..5df95abf3 100644--- a/src/locales/es/translation.json+++ b/src/locales/es/translation.json@@ -1755,7 +1755,14 @@"them": "Ellos","label": "Participante {{n}}","identifyingSpeakers": "Identificando participantes...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "Escribe un nombre",+ "knownSpeakers": "Hablantes conocidos",+ "meetingAttendees": "Asistentes de la reunión",+ "assignName": "Asignar nombre",+ "isThisPerson": "¿Es {{name}}?",+ "confirmed": "Confirmado",+ "nameAssigned": "Nombre asignado"}},"dictionary": {diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.jsonindex e9b55686e..1a2f60d9d 100644--- a/src/locales/fr/translation.json+++ b/src/locales/fr/translation.json@@ -1816,7 +1816,14 @@"them": "Eux","label": "Intervenant {{n}}","identifyingSpeakers": "Identification des intervenants...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "Saisir un nom",+ "knownSpeakers": "Interlocuteurs connus",+ "meetingAttendees": "Participants à la réunion",+ "assignName": "Attribuer un nom",+ "isThisPerson": "Est-ce {{name}} ?",+ "confirmed": "Confirmé",+ "nameAssigned": "Nom attribué"}},"dictionary": {diff --git a/src/locales/it/translation.json b/src/locales/it/translation.jsonindex e18bb7c56..4dc5071b3 100644--- a/src/locales/it/translation.json+++ b/src/locales/it/translation.json@@ -1755,7 +1755,14 @@"them": "Loro","label": "Partecipante {{n}}","identifyingSpeakers": "Identificazione dei partecipanti...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "Digita un nome",+ "knownSpeakers": "Parlanti noti",+ "meetingAttendees": "Partecipanti alla riunione",+ "assignName": "Assegna nome",+ "isThisPerson": "È {{name}}?",+ "confirmed": "Confermato",+ "nameAssigned": "Nome assegnato"}},"dictionary": {diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.jsonindex c55e23da3..b8fd9be88 100644--- a/src/locales/ja/translation.json+++ b/src/locales/ja/translation.json@@ -1683,7 +1683,14 @@"them": "相手","label": "話者 {{n}}","identifyingSpeakers": "話者を識別中...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "名前を入力",+ "knownSpeakers": "登録済みの話者",+ "meetingAttendees": "会議の参加者",+ "assignName": "名前を割り当て",+ "isThisPerson": "{{name}}さんですか?",+ "confirmed": "確認済み",+ "nameAssigned": "名前を割り当てました"}},"dictionary": {diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.jsonindex 6e230841b..ed60065de 100644--- a/src/locales/pt/translation.json+++ b/src/locales/pt/translation.json@@ -1755,7 +1755,14 @@"them": "Outro","label": "Participante {{n}}","identifyingSpeakers": "Identificando participantes...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "Digite um nome",+ "knownSpeakers": "Falantes conhecidos",+ "meetingAttendees": "Participantes da reunião",+ "assignName": "Atribuir nome",+ "isThisPerson": "É {{name}}?",+ "confirmed": "Confirmado",+ "nameAssigned": "Nome atribuído"}},"dictionary": {diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.jsonindex 5bd82d74c..a61edeb37 100644--- a/src/locales/ru/translation.json+++ b/src/locales/ru/translation.json@@ -1755,7 +1755,14 @@"them": "Собеседник","label": "Участник {{n}}","identifyingSpeakers": "Определение участников...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "Введите имя",+ "knownSpeakers": "Известные собеседники",+ "meetingAttendees": "Участники встречи",+ "assignName": "Назначить имя",+ "isThisPerson": "Это {{name}}?",+ "confirmed": "Подтверждено",+ "nameAssigned": "Имя назначено"}},"dictionary": {diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.jsonindex 3201baec9..bbf29bddd 100644--- a/src/locales/zh-CN/translation.json+++ b/src/locales/zh-CN/translation.json@@ -1755,7 +1755,14 @@"them": "对方","label": "发言者 {{n}}","identifyingSpeakers": "正在识别发言者...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "输入姓名",+ "knownSpeakers": "已知说话者",+ "meetingAttendees": "会议参与者",+ "assignName": "分配姓名",+ "isThisPerson": "这是{{name}}吗?",+ "confirmed": "已确认",+ "nameAssigned": "姓名已分配"}},"dictionary": {diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.jsonindex 0bd9feb21..4d61a971c 100644--- a/src/locales/zh-TW/translation.json+++ b/src/locales/zh-TW/translation.json@@ -1755,7 +1755,14 @@"them": "對方","label": "發言者 {{n}}","identifyingSpeakers": "正在辨識發言者...",- "speakersIdentified": "Speakers identified"+ "speakersIdentified": "Speakers identified",+ "typeNamePlaceholder": "輸入姓名",+ "knownSpeakers": "已知發言者",+ "meetingAttendees": "會議參與者",+ "assignName": "指派姓名",+ "isThisPerson": "這是{{name}}嗎?",+ "confirmed": "已確認",+ "nameAssigned": "姓名已指派"}},"dictionary": {diff --git a/src/types/electron.ts b/src/types/electron.tsindex 15f8b9c49..29b246e67 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -1330,10 +1330,42 @@ declare global {source: "mic" | "system";timestamp?: number;speaker?: string;+ speakerName?: string;+ suggestedName?: string;+ suggestedProfileId?: number;}>;+ speakerEmbeddings?: Record<string, number[]> | null;}) => void) => () => void;+ // Speaker name mapping+ getSpeakerMappings?: (noteId: number) => Promise<Array<{+ note_id: number;+ speaker_id: string;+ profile_id: number | null;+ display_name: string;+ }>>;+ setSpeakerMapping?: (+ noteId: number,+ speakerId: string,+ displayName: string,+ email?: string | null,+ profileId?: number | null+ ) => Promise<{ success: boolean; profileId: number | null }>;+ removeSpeakerMapping?: (noteId: number, speakerId: string) => Promise<{ success: boolean }>;+ getSpeakerProfiles?: () => Promise<Array<{+ id: number;+ display_name: string;+ email: string | null;+ sample_count: number;+ created_at: string;+ updated_at: string;+ }>>;+ saveNoteSpeakerEmbeddings?: (+ noteId: number,+ embeddings: Record<string, number[]>+ ) => Promise<{ success: boolean }>;+// Dictation realtime streamingdictationRealtimeWarmup?: (options: {model?: string;diff --git a/src/utils/parseTranscriptSegments.ts b/src/utils/parseTranscriptSegments.tsindex e9ffce1ff..60e97bd46 100644--- a/src/utils/parseTranscriptSegments.ts+++ b/src/utils/parseTranscriptSegments.ts@@ -8,6 +8,9 @@ export function parseTranscriptSegments(raw: string): TranscriptSegment[] {source: "mic" | "system";timestamp?: number;speaker?: string;+ speakerName?: string;+ suggestedName?: string;+ suggestedProfileId?: number;}>;return parsed.map((s, i) => ({id: `stored-${i}`,@@ -15,6 +18,9 @@ export function parseTranscriptSegments(raw: string): TranscriptSegment[] {source: s.source,timestamp: s.timestamp,speaker: s.speaker,+ speakerName: s.speakerName,+ suggestedName: s.suggestedName,+ suggestedProfileId: s.suggestedProfileId,}));} catch (e) {logger.warn("Failed to parse transcript segments", e);