Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(diarization): stop phantom speakers in uploaded recordings (#2403)
void13afd03e54d6
diff --git a/src/components/notes/UploadAudioView.tsx b/src/components/notes/UploadAudioView.tsxindex b2a8265fe..e801133d4 100644--- a/src/components/notes/UploadAudioView.tsx+++ b/src/components/notes/UploadAudioView.tsx@@ -189,9 +189,11 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploaconst [diarizationEnabled, setDiarizationEnabled] = useState(() => localStorage.getItem("uploadDiarizationEnabled") === "true");- const [diarizationNumSpeakers, setDiarizationNumSpeakers] = useState<string>(- () => localStorage.getItem("uploadDiarizationNumSpeakers") || ""- );+ // Earlier builds accepted decimals; a saved one would read as Auto.+ const [diarizationNumSpeakers, setDiarizationNumSpeakers] = useState<string>(() => {+ const saved = localStorage.getItem("uploadDiarizationNumSpeakers") || "";+ return /^\d+$/.test(saved) ? saved : "";+ });const [diarizationModelsReady, setDiarizationModelsReady] = useState<boolean | null>(null);const [diarizationDownloading, setDiarizationDownloading] = useState(false);@@ -1319,15 +1321,21 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploatype="number"min="2"max={MAX_SPEAKER_COUNT}+ step="1"+ inputMode="numeric"value={diarizationNumSpeakers}+ onKeyDown={(e) => {+ if ([".", ",", "e", "E", "+", "-"].includes(e.key)) e.preventDefault();+ }}onChange={(e) => {const raw = e.target.value;if (raw === "") {setDiarizationNumSpeakers("");return;}- const n = Math.max(2, Math.min(MAX_SPEAKER_COUNT, Number(raw)));- setDiarizationNumSpeakers(String(isNaN(n) ? "" : n));+ const n = Number(raw);+ if (!Number.isInteger(n)) return;+ setDiarizationNumSpeakers(String(Math.max(2, Math.min(MAX_SPEAKER_COUNT, n))));}}placeholder={t("notes.upload.numSpeakersPlaceholder")}className="h-9 w-32 shrink-0 rounded-xl! px-3 text-sm placeholder:text-foreground/45 [appearance:textfield] [&::-webkit-outer-spin-button]:appearance-none [&::-webkit-inner-spin-button]:appearance-none"diff --git a/src/helpers/diarizationPolicy.js b/src/helpers/diarizationPolicy.jsindex 39c47c2c1..1a587a7ce 100644--- a/src/helpers/diarizationPolicy.js+++ b/src/helpers/diarizationPolicy.js@@ -6,8 +6,10 @@// speaker's embeddings drift wider than that and agglomerative clustering// stops merging early — a 73-minute voice memo produced 46 "speakers". Ramp// the threshold with duration instead of pretending one constant fits both.+// The top stays below 0.72: from there up, a two-person Spanish call merged+// both speakers into one cluster.const DEFAULT_CLUSTER_THRESHOLD = 0.55;-const LONG_AUDIO_CLUSTER_THRESHOLD = 0.8;+const LONG_AUDIO_CLUSTER_THRESHOLD = 0.7;const THRESHOLD_RAMP_START_SECONDS = 15 * 60;const THRESHOLD_RAMP_END_SECONDS = 60 * 60;@@ -16,6 +18,31 @@ const THRESHOLD_RAMP_END_SECONDS = 60 * 60;// at least one sentence, so each blip becomes a phantom [Speaker N].const MIN_CLUSTER_TOTAL_SECONDS = 1;+// Longer recordings grow phantoms well past that floor: backchannels and turn+// starts whose 1–3 s embeddings land near neither speaker's centroid (a+// 20-minute two-person call produced four, holding 8 % of the speech). A+// phantom is both a sliver of the speech and made of short segments; a real+// minor speaker holds longer turns. Known cost: a real third participant with+// under 10 % of the speech in turns averaging under 3 s looks the same and is+// dropped too. Only their voice tells them apart, which this rule doesn't use.+const PHANTOM_MAX_SHARE = 0.1;+const PHANTOM_MAX_MEAN_SEGMENT_SECONDS = 3;+// Merging two real speakers is the worse failure, so the two largest clusters+// are never treated as phantoms.+const PROTECTED_CLUSTER_COUNT = 2;++// When clustering merges two real speakers, one cluster ends up with nearly+// all the speech beside phantoms of backchannels and turn starts: on every+// collapsed run of #2021's call the top held 94–96 % and each phantom+// averaged 1.7–2 s segments. Someone who asks questions, however short,+// averages longer segments, so an interview or a lecture with audience+// questions keeps its labels. Known cost: a listener who only backchannels+// beside a speaker holding over 93 % looks the same, and that run loses its+// labels. Clusters under the 1 s floor are noise and don't count, so a+// monologue with a stray blip isn't a collapse.+const COLLAPSED_TOP_SHARE = 0.93;+const COLLAPSED_MAX_MEAN_SEGMENT_SECONDS = 2.2;+function clusterThresholdForDuration(durationSeconds) {if (!Number.isFinite(durationSeconds) || durationSeconds <= THRESHOLD_RAMP_START_SECONDS) {return DEFAULT_CLUSTER_THRESHOLD;@@ -38,19 +65,85 @@ function resolveClusterThreshold(durationSeconds, requestedThreshold) {return Math.min(1, Math.max(0, parsedThreshold));}-function dropNegligibleClusters(segments, minTotalSeconds = MIN_CLUSTER_TOTAL_SECONDS) {- if (!segments?.length) return segments;- const totals = new Map();+function summarizeClusters(segments) {+ const clusters = new Map();+ let totalSpeech = 0;for (const s of segments) {- totals.set(s.speaker, (totals.get(s.speaker) || 0) + (s.end - s.start));+ const cluster = clusters.get(s.speaker) || { total: 0, count: 0 };+ cluster.total += s.end - s.start;+ cluster.count += 1;+ clusters.set(s.speaker, cluster);+ totalSpeech += s.end - s.start;}+ return { clusters, totalSpeech };+}++function dropNegligibleClusters(segments) {+ if (!segments?.length) return segments;+ const { clusters, totalSpeech } = summarizeClusters(segments);const keep = new Set(- [...totals].filter(([, total]) => total >= minTotalSeconds).map(([speaker]) => speaker)+ [...clusters]+ .sort((a, b) => b[1].total - a[1].total)+ .filter(([, { total, count }], rank) => {+ if (total < MIN_CLUSTER_TOTAL_SECONDS) return false;+ return (+ rank < PROTECTED_CLUSTER_COUNT ||+ total / totalSpeech >= PHANTOM_MAX_SHARE ||+ total / count >= PHANTOM_MAX_MEAN_SEGMENT_SECONDS+ );+ })+ .map(([speaker]) => speaker));- if (keep.size === 0 || keep.size === totals.size) return segments;+ if (keep.size === 0 || keep.size === clusters.size) return segments;return segments.filter((s) => keep.has(s.speaker));}+function isCollapsedDiarization(segments) {+ const { clusters } = summarizeClusters(segments);+ const heard = [...clusters.values()].filter(({ total }) => total >= MIN_CLUSTER_TOTAL_SECONDS);+ if (heard.length < 2) return false;+ const [largest, ...rest] = heard.sort((a, b) => b.total - a.total);+ const heardSpeech = heard.reduce((sum, { total }) => sum + total, 0);+ return (+ largest.total / heardSpeech > COLLAPSED_TOP_SHARE &&+ rest.every(({ total, count }) => total / count < COLLAPSED_MAX_MEAN_SEGMENT_SECONDS)+ );+}++// Keeps the `cap` largest clusters and relabels each extra one to the kept+// speaker whose voice centroid it is most similar to. Folding extras into the+// largest cluster (capSpeakerClusters) handed one person's turns to the other+// whenever clustering split a real speaker. Clusters without a centroid still+// fold into the largest.+function capSpeakerClustersByVoice(segments, cap, centroids, similarity) {+ const { clusters } = summarizeClusters(segments);+ if (clusters.size <= cap) return segments;++ const ranked = [...clusters].sort((a, b) => b[1].total - a[1].total).map(([speaker]) => speaker);+ const kept = ranked.slice(0, cap);+ const relabel = new Map();+ for (const extra of ranked.slice(cap)) {+ const voice = centroids.get(extra);+ let target = kept[0];+ let bestSimilarity = -Infinity;+ if (voice) {+ for (const speaker of kept) {+ const keptVoice = centroids.get(speaker);+ if (!keptVoice) continue;+ const score = similarity(voice, keptVoice);+ if (score > bestSimilarity) {+ bestSimilarity = score;+ target = speaker;+ }+ }+ }+ relabel.set(extra, target);+ }+ return segments.map((s) =>+ relabel.has(s.speaker) ? { ...s, speaker: relabel.get(s.speaker) } : s+ );+}+module.exports = {DEFAULT_CLUSTER_THRESHOLD,LONG_AUDIO_CLUSTER_THRESHOLD,@@ -60,4 +153,6 @@ module.exports = {clusterThresholdForDuration,resolveClusterThreshold,dropNegligibleClusters,+ isCollapsedDiarization,+ capSpeakerClustersByVoice,};diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex fad3de9c2..27ab76f0b 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -3777,14 +3777,16 @@ class IPCHandlers {if (!realPath) return { success: false, error: "File path not allowed" };filePath = realPath;- const numSpeakers = Math.min(- MAX_SPEAKER_COUNT,- Math.max(-1, Math.round(Number(options.numSpeakers) || -1))- );+ const maxSpeakers = normalizeStoredSpeakerCount(options.numSpeakers) ?? MAX_SPEAKER_COUNT;const { convertToWav } = require("./ffmpegUtils");const { getSafeTempDir } = require("./safeTempDir");- const { resolveClusterThreshold, dropNegligibleClusters } = require("./diarizationPolicy");+ const {+ resolveClusterThreshold,+ dropNegligibleClusters,+ isCollapsedDiarization,+ capSpeakerClustersByVoice,+ } = require("./diarizationPolicy");const { PCM16_MONO_16K_BYTES_PER_SECOND } = require("./transcriptionTimeout");const wavPath = path.join(getSafeTempDir(), `ow-diarize-${Date.now()}.wav`);@@ -3798,22 +3800,48 @@ class IPCHandlers {const durationSeconds = fs.statSync(wavPath).size / PCM16_MONO_16K_BYTES_PER_SECOND;const threshold = resolveClusterThreshold(durationSeconds, options.threshold);- let segments = await this.diarizationManager.diarize(wavPath, {- numSpeakers,- threshold,- signal,- });+ // Always auto-cluster: forcing 2 clusters on #2021's two-person call+ // split short utterances from long ones instead of one speaker from+ // the other, merging both speakers. The requested count is applied+ // as a cap after cleanup instead.+ let segments = await this.diarizationManager.diarize(wavPath, { threshold, signal });if (signal?.aborted) {return { success: false, error: "Cancelled", code: "UPLOAD_CANCELLED" };}+ // A collapsed run's labels are wrong either way: two people under+ // one label, or one person split off into a phantom. With no+ // segments the renderer keeps the plain transcript and shows the+ // diarization warning instead. Judged before phantoms are dropped;+ // a request for one speaker is met by the cap below instead.+ if (maxSpeakers > 1 && isCollapsedDiarization(segments)) {+ debugLogger.warn("Discarding diarization: one cluster holds nearly all speech", {+ durationSeconds,+ });+ segments = [];+ }// The meeting path caps clusters via its expectation resolver; this// path fed raw sherpa output to the merge, which is how a 2-person// voice memo surfaced 46 speakers.segments = dropNegligibleClusters(segments);- segments = this.diarizationManager.capSpeakerClusters(- segments,- numSpeakers > 0 ? numSpeakers : MAX_SPEAKER_COUNT- );+ if (new Set(segments.map((s) => s.speaker)).size > maxSpeakers) {+ const speakerEmbeddings = require("./speakerEmbeddings");+ let centroids = new Map();+ try {+ centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments, {+ signal,+ });+ } catch (error) {+ debugLogger.warn("Speaker voices unavailable; extra clusters fold into the largest", {+ error: error.message,+ });+ }+ if (signal?.aborted) {+ return { success: false, error: "Cancelled", code: "UPLOAD_CANCELLED" };+ }+ segments = capSpeakerClustersByVoice(segments, maxSpeakers, centroids, (a, b) =>+ speakerEmbeddings.cosineSimilarity(a, b)+ );+ }// Callers persist this as audio_duration_seconds: for picked files// the renderer has no other duration source.return { success: true, segments, durationSeconds };diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.jsindex 736bed0dd..3103da62e 100644--- a/src/helpers/speakerEmbeddings.js+++ b/src/helpers/speakerEmbeddings.js@@ -10,6 +10,11 @@ const MIN_SEGMENT_SECONDS = 1.5;const MIN_SEGMENT_SAMPLES = SAMPLE_RATE * MIN_SEGMENT_SECONDS;const MAX_EMBEDDING_SECONDS = 8;const MAX_EMBEDDING_SAMPLES = SAMPLE_RATE * MAX_EMBEDDING_SECONDS;+const BYTES_PER_SAMPLE = 2;+// Same as the meeting path's per-speaker centroids.+const CENTROID_SEGMENTS_PER_CLUSTER = 3;+// Enough for the RIFF, fmt and metadata chunks ffmpeg writes before "data".+const WAV_HEADER_READ_BYTES = 64 * 1024;const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";// Live meetings extract during remote speech, so this normally fires once a meeting and its// post-meeting diarization are done (a longer silence mid-meeting costs one reload). The@@ -133,26 +138,85 @@ class SpeakerEmbeddings {if (endSec - startSec < MIN_SEGMENT_SECONDS) return null;const buf = fs.readFileSync(wavPath);- const { sampleRate, dataOffset } = this._parseWavHeader(buf);+ const { byteOffset, numSamples } = this._embeddingWindow(+ this._parseWavHeader(buf),+ startSec,+ endSec+ );+ return this._extractEmbeddingFromSamples(this._decodePcm16(buf, byteOffset, numSamples));+ }++ // One voice per diarized cluster, from its longest segments. Reads only the+ // audio each embedding uses: an upload's WAV runs about 115 MB an hour. A+ // cluster whose extraction fails goes without a voice; the others keep theirs.+ async extractClusterCentroids(wavPath, segments, { signal = null } = {}) {+ const bySpeaker = new Map();+ for (const segment of segments) {+ if (segment.end - segment.start < MIN_SEGMENT_SECONDS) continue;+ if (!bySpeaker.has(segment.speaker)) bySpeaker.set(segment.speaker, []);+ bySpeaker.get(segment.speaker).push(segment);+ }++ const centroids = new Map();+ const fd = fs.openSync(wavPath, "r");+ try {+ const header = this._parseWavHeader(this._readBytes(fd, 0, WAV_HEADER_READ_BYTES));+ for (const [speaker, speakerSegments] of bySpeaker) {+ if (signal?.aborted) break;+ const longest = speakerSegments+ .sort((a, b) => b.end - b.start - (a.end - a.start))+ .slice(0, CENTROID_SEGMENTS_PER_CLUSTER);+ try {+ const embeddings = [];+ for (const segment of longest) {+ const { byteOffset, numSamples } = this._embeddingWindow(+ header,+ segment.start,+ segment.end+ );+ const pcm = this._readBytes(fd, byteOffset, numSamples * BYTES_PER_SAMPLE);+ const embedding = await this._extractEmbeddingFromSamples(+ this._decodePcm16(pcm, 0, numSamples)+ );+ if (embedding) embeddings.push(embedding);+ }+ if (embeddings.length > 0) centroids.set(speaker, this.computeCentroid(embeddings));+ } catch (error) {+ debugLogger.warn("speaker-embeddings cluster voice unavailable", {+ speaker,+ error: error.message,+ });+ }+ }+ } finally {+ fs.closeSync(fd);+ }+ return centroids;+ }++ _readBytes(fd, position, length) {+ const buf = Buffer.alloc(length);+ const bytesRead = fs.readSync(fd, buf, 0, length, position);+ return buf.subarray(0, bytesRead);+ }+ // The last MAX_EMBEDDING_SECONDS of the segment, as a byte offset into the WAV.+ _embeddingWindow({ sampleRate, dataOffset }, startSec, endSec) {const cappedSeconds = Math.min(endSec - startSec, MAX_EMBEDDING_SECONDS);- const cappedStartSec = endSec - cappedSeconds;- const startSample = Math.floor(cappedStartSec * sampleRate);- const endSample = Math.floor(endSec * sampleRate);- const numSamples = endSample - startSample;+ const startSample = Math.floor((endSec - cappedSeconds) * sampleRate);+ const numSamples = Math.floor(endSec * sampleRate) - startSample;+ return { byteOffset: dataOffset + startSample * BYTES_PER_SAMPLE, numSamples };+ }+ // Samples past the end of buf stay silent.+ _decodePcm16(buf, offset, numSamples) {const samples = new Float32Array(numSamples);- const bytesPerSample = 2;- const offset = dataOffset + startSample * bytesPerSample;-for (let i = 0; i < numSamples; i++) {- const bytePos = offset + i * bytesPerSample;+ const bytePos = offset + i * BYTES_PER_SAMPLE;if (bytePos + 1 >= buf.length) break;- const int16 = buf.readInt16LE(bytePos);- samples[i] = int16 / 32768;+ samples[i] = buf.readInt16LE(bytePos) / 32768;}-- return this._extractEmbeddingFromSamples(samples);+ return samples;}_parseWavHeader(buf) {diff --git a/src/locales/ar/translation.json b/src/locales/ar/translation.jsonindex c174dcf03..82da25580 100644--- a/src/locales/ar/translation.json+++ b/src/locales/ar/translation.json@@ -2231,7 +2231,7 @@"speakerDetectionDescription": "تحديد المتحدثين المختلفين في الصوت","numSpeakersPlaceholder": "اكتشاف تلقائي","numSpeakersLabel": "عدد المتحدثين",- "numSpeakersHint": "يؤدي تحديد العدد الدقيق إلى تحسين الدقة، خاصة في التسجيلات الطويلة.",+ "numSpeakersHint": "يُكتشف المتحدثون تلقائيًا، بحد أقصى هذا العدد.","downloadingModels": "جاري تنزيل النماذج...","openaiDiarizeNote": "يستبدل نموذج التفريغ الذي اخترته بالنموذج gpt-4o-transcribe-diarize، وهو نموذج OpenAI الوحيد الذي يدعم اكتشاف المتحدثين. قد تختلف التكاليف عن نموذجك المحدد.","mistralDiarizeNote": "تم تضمين ميزة اكتشاف المتحدث بدون تكلفة إضافية مع Voxtral.",diff --git a/src/locales/de/translation.json b/src/locales/de/translation.jsonindex c89939715..406a0a76f 100644--- a/src/locales/de/translation.json+++ b/src/locales/de/translation.json@@ -3433,7 +3433,7 @@"speakerDetectionDescription": "Verschiedene Sprecher im Audio identifizieren","numSpeakersPlaceholder": "Automatisch erkennen","numSpeakersLabel": "Anzahl der Sprecher",- "numSpeakersHint": "Die genaue Anzahl verbessert die Genauigkeit, besonders bei langen Aufnahmen.",+ "numSpeakersHint": "Sprecher werden automatisch erkannt, bis zu dieser Anzahl.","downloadingModels": "Modelle werden heruntergeladen...","openaiDiarizeNote": "Überschreibt Ihr Transkriptionsmodell mit gpt-4o-transcribe-diarize, dem einzigen OpenAI-Modell mit Sprechererkennung. Die Kosten können von Ihrem gewählten Modell abweichen.","mistralDiarizeNote": "Sprechererkennung ohne Zusatzkosten mit Voxtral enthalten.",diff --git a/src/locales/en/translation.json b/src/locales/en/translation.jsonindex 1dcdb078e..7990787a0 100644--- a/src/locales/en/translation.json+++ b/src/locales/en/translation.json@@ -3433,7 +3433,7 @@"speakerDetectionDescription": "Identify different speakers in the audio","numSpeakersPlaceholder": "Auto-detect","numSpeakersLabel": "Number of speakers",- "numSpeakersHint": "Setting the exact count improves accuracy, especially on long recordings.",+ "numSpeakersHint": "Speakers are detected automatically, up to this number.","downloadingModels": "Downloading models...","openaiDiarizeNote": "Overrides your transcription model with gpt-4o-transcribe-diarize, the only OpenAI model with speaker detection. Costs may differ from your selected model.","mistralDiarizeNote": "Speaker detection included at no extra cost with Voxtral.",diff --git a/src/locales/es/translation.json b/src/locales/es/translation.jsonindex 4c7835468..90e24f75d 100644--- a/src/locales/es/translation.json+++ b/src/locales/es/translation.json@@ -3134,7 +3134,7 @@"speakerDetectionDescription": "Identificar diferentes hablantes en el audio","numSpeakersPlaceholder": "Detección automática","numSpeakersLabel": "Número de hablantes",- "numSpeakersHint": "Indicar el número exacto mejora la precisión, sobre todo en grabaciones largas.",+ "numSpeakersHint": "Los hablantes se detectan automáticamente, hasta este número.","downloadingModels": "Descargando modelos...","openaiDiarizeNote": "Reemplaza tu modelo de transcripción con gpt-4o-transcribe-diarize, el único modelo de OpenAI con detección de hablantes. Los costos pueden diferir de tu modelo seleccionado.","mistralDiarizeNote": "Detección de hablantes incluida sin costo adicional con Voxtral.",diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.jsonindex 2bd29242f..8aaeaf708 100644--- a/src/locales/fr/translation.json+++ b/src/locales/fr/translation.json@@ -3433,7 +3433,7 @@"speakerDetectionDescription": "Identifier les différents intervenants dans l'audio","numSpeakersPlaceholder": "Détection automatique","numSpeakersLabel": "Nombre d'intervenants",- "numSpeakersHint": "Indiquer le nombre exact améliore la précision, surtout pour les enregistrements longs.",+ "numSpeakersHint": "Les intervenants sont détectés automatiquement, jusqu'à ce nombre.","downloadingModels": "Téléchargement des modèles...","openaiDiarizeNote": "Remplace votre modèle de transcription par gpt-4o-transcribe-diarize, le seul modèle OpenAI avec détection des locuteurs. Les coûts peuvent différer de votre modèle sélectionné.","mistralDiarizeNote": "Détection des locuteurs incluse sans frais supplémentaires avec Voxtral.",diff --git a/src/locales/it/translation.json b/src/locales/it/translation.jsonindex 323dff24d..458c8245d 100644--- a/src/locales/it/translation.json+++ b/src/locales/it/translation.json@@ -3081,7 +3081,7 @@"speakerDetectionDescription": "Identifica i diversi parlanti nell'audio","numSpeakersPlaceholder": "Rilevamento automatico","numSpeakersLabel": "Numero di parlanti",- "numSpeakersHint": "Indicare il numero esatto migliora la precisione, soprattutto nelle registrazioni lunghe.",+ "numSpeakersHint": "I parlanti vengono rilevati automaticamente, fino a questo numero.","downloadingModels": "Download modelli in corso...","openaiDiarizeNote": "Sostituisce il tuo modello di trascrizione con gpt-4o-transcribe-diarize, l’unico modello OpenAI con rilevamento dei parlanti. I costi potrebbero differire dal modello selezionato.","mistralDiarizeNote": "Rilevamento dei parlanti incluso senza costi aggiuntivi con Voxtral.",diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.jsonindex 2292c8c04..fe012dc53 100644--- a/src/locales/ja/translation.json+++ b/src/locales/ja/translation.json@@ -3148,7 +3148,7 @@"speakerDetectionDescription": "音声内の異なる話者を識別","numSpeakersPlaceholder": "自動検出","numSpeakersLabel": "話者の人数",- "numSpeakersHint": "正確な人数を設定すると、特に長い録音で精度が向上します。",+ "numSpeakersHint": "話者はこの人数を上限として自動で検出されます。","downloadingModels": "モデルをダウンロード中...","openaiDiarizeNote": "話者検出をサポートする唯一のOpenAIモデルであるgpt-4o-transcribe-diarizeに切り替わります。選択したモデルとコストが異なる場合があります。","mistralDiarizeNote": "Voxtralでの話者検出は追加料金なしで利用できます。",diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.jsonindex 985e120a5..a67737809 100644--- a/src/locales/pt/translation.json+++ b/src/locales/pt/translation.json@@ -3081,7 +3081,7 @@"speakerDetectionDescription": "Identificar diferentes locutores no áudio","numSpeakersPlaceholder": "Detecção automática","numSpeakersLabel": "Número de falantes",- "numSpeakersHint": "Definir o número exato melhora a precisão, principalmente em gravações longas.",+ "numSpeakersHint": "Os falantes são detectados automaticamente, até este número.","downloadingModels": "Baixando modelos...","openaiDiarizeNote": "Substitui seu modelo de transcrição por gpt-4o-transcribe-diarize, o único modelo OpenAI com detecção de falantes. Os custos podem diferir do modelo selecionado.","mistralDiarizeNote": "Detecção de falantes incluída sem custo adicional com Voxtral.",diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.jsonindex c2e198291..c7e06fb54 100644--- a/src/locales/ru/translation.json+++ b/src/locales/ru/translation.json@@ -3101,7 +3101,7 @@"speakerDetectionDescription": "Определить разных говорящих в аудио","numSpeakersPlaceholder": "Автоопределение","numSpeakersLabel": "Количество говорящих",- "numSpeakersHint": "Точное число повышает точность, особенно для длинных записей.",+ "numSpeakersHint": "Говорящие определяются автоматически, но не больше этого числа.","downloadingModels": "Загрузка моделей...","openaiDiarizeNote": "Заменяет вашу модель транскрипции на gpt-4o-transcribe-diarize — единственную модель OpenAI с определением говорящих. Стоимость может отличаться от выбранной модели.","mistralDiarizeNote": "Определение говорящих включено без дополнительной оплаты с Voxtral.",diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.jsonindex fd4e8de1a..562eaa93d 100644--- a/src/locales/zh-CN/translation.json+++ b/src/locales/zh-CN/translation.json@@ -3081,7 +3081,7 @@"speakerDetectionDescription": "识别音频中的不同说话人","numSpeakersPlaceholder": "自动检测","numSpeakersLabel": "说话人数",- "numSpeakersHint": "设置准确的人数可以提高准确性,尤其是长录音。",+ "numSpeakersHint": "将自动检测说话人,不超过此人数。","downloadingModels": "正在下载模型...","openaiDiarizeNote": "将替换您的转录模型为 gpt-4o-transcribe-diarize,这是唯一支持说话人检测的 OpenAI 模型。费用可能与您选择的模型不同。","mistralDiarizeNote": "使用 Voxtral 时说话人检测无需额外费用。",diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.jsonindex 5ec7d8394..f563d953f 100644--- a/src/locales/zh-TW/translation.json+++ b/src/locales/zh-TW/translation.json@@ -3081,7 +3081,7 @@"speakerDetectionDescription": "辨識音訊中的不同說話者","numSpeakersPlaceholder": "自動偵測","numSpeakersLabel": "說話人數",- "numSpeakersHint": "設定準確的人數可以提高準確度,尤其是長錄音。",+ "numSpeakersHint": "將自動偵測說話人,不超過此人數。","downloadingModels": "正在下載模型...","openaiDiarizeNote": "將替換您的轉錄模型為 gpt-4o-transcribe-diarize,這是唯一支援說話人偵測的 OpenAI 模型。費用可能與您選擇的模型不同。","mistralDiarizeNote": "使用 Voxtral 時說話人偵測無需額外費用。",diff --git a/src/services/uploadNotes.ts b/src/services/uploadNotes.tsindex ac68bd4ff..d21225c55 100644--- a/src/services/uploadNotes.ts+++ b/src/services/uploadNotes.ts@@ -1,16 +1,5 @@-import { MAX_SPEAKER_COUNT } from "../constants/speakerDetection.json";import type { DiarizationSettings } from "./fileTranscription";-// Renderer twin of normalizeStoredSpeakerCount in ../helpers/speakerCount.js.-// That file is CommonJS for the main process (database.js, ipcHandlers.js), and-// renderer source cannot load CJS modules under Vite — test/services/-// uploadNotes.test.js holds the two implementations to identical outputs.-function normalizeSpeakerCount(value: number | null | undefined): number | null {- const count = Number(value);- if (!Number.isInteger(count) || count < 1) return null;- return Math.min(count, MAX_SPEAKER_COUNT);-}-export interface UploadSegment {text: string;start: number;@@ -44,15 +33,14 @@ export function buildUploadTranscript(return stored.length ? JSON.stringify(stored) : undefined;}-// What an upload persists onto the note row about the diarizer invocation,+// What an upload persists onto the note row about the diarization run,// matching the meeting path's write semantics: the columns are written only// when diarization ran. A null diarization_enabled means "user never chose" and// consumers fall back to the global speaker setting — writing 0 would force-// diarization off when recording into the note later. The-// expected_speaker_count is only ever a count the diarizer was actually invoked-// with (numSpeakers lingers in localStorage while its input is hidden); auto-// detection stays null so isExplicitSpeakerCount never mistakes it for an-// explicit choice.+// diarization off when recording into the note later. The speaker count the+// user entered is never stored: the upload applied it as a cap on detected+// speakers, but recording into the note forces expected_speaker_count as the+// exact count, so a generous cap would split that meeting into phantoms.export function buildUploadNoteMetadata(diarization: DiarizationSettings,durationSeconds?: number | null,@@ -63,12 +51,7 @@ export function buildUploadNoteMetadata(const noteUpdates: Record<string, unknown> | null =diarization.enabled || transcript? {- ...(diarization.enabled- ? {- diarization_enabled: 1,- expected_speaker_count: normalizeSpeakerCount(diarization.numSpeakers),- }- : {}),+ ...(diarization.enabled ? { diarization_enabled: 1 } : {}),...(transcript ? { transcript } : {}),}: null;diff --git a/test/helpers/diarizationPolicy.test.js b/test/helpers/diarizationPolicy.test.jsindex 0573566aa..6243ac614 100644--- a/test/helpers/diarizationPolicy.test.js+++ b/test/helpers/diarizationPolicy.test.js@@ -10,6 +10,8 @@ const {clusterThresholdForDuration,resolveClusterThreshold,dropNegligibleClusters,+ isCollapsedDiarization,+ capSpeakerClustersByVoice,} = require("../../src/helpers/diarizationPolicy");// 0.55 is tuned for short clean audio; on a 73-minute single-mic voice memo one@@ -42,6 +44,12 @@ test("hour-plus audio is clamped to the long-audio threshold", () => {assert.equal(clusterThresholdForDuration(3 * 3600), LONG_AUDIO_CLUSTER_THRESHOLD);});+// Too high a threshold is the opposite failure: at 0.72 and above, a+// 20-minute two-person call (#2021) came back as one speaker plus phantoms.+test("the long-audio threshold stays below where #2021's two speakers merged", () => {+ assert.ok(clusterThresholdForDuration(3 * 3600) < 0.72);+});+test("unknown duration falls back to the default threshold", () => {assert.equal(clusterThresholdForDuration(NaN), DEFAULT_CLUSTER_THRESHOLD);assert.equal(clusterThresholdForDuration(undefined), DEFAULT_CLUSTER_THRESHOLD);@@ -73,7 +81,7 @@ test("clusters below the minimum total speaking time are dropped", () => {);});-test("a cluster whose short segments add up past the minimum survives", () => {+test("the second-largest cluster survives once its short segments pass the minimum", () => {const segments = [{ start: 0, end: 30, speaker: "speaker_0" },{ start: 30, end: 30.6, speaker: "speaker_1" },@@ -82,6 +90,95 @@ test("a cluster whose short segments add up past the minimum survives", () => {assert.equal(dropNegligibleClusters(segments).length, 3);});+// `count` segments of `seconds` each, so a fixture can reproduce a cluster's+// measured total speech and mean segment length.+function cluster(speaker, count, seconds) {+ return Array.from({ length: count }, (_, i) => ({+ start: i * 20,+ end: i * 20 + seconds,+ speaker,+ }));+}++const speakersOf = (segments) => [...new Set(segments.map((s) => s.speaker))].sort();++// Measured on a 20-minute two-person Spanish call (#2021): four phantom+// clusters of short backchannels and turn starts, each over the 1 s floor.+test("short-utterance phantom clusters are dropped from a two-person call", () => {+ const segments = [+ ...cluster("speaker_06", 79, 6.46),+ ...cluster("speaker_01", 85, 5.44),+ ...cluster("speaker_00", 23, 1.72),+ ...cluster("speaker_02", 20, 1.57),+ ...cluster("speaker_04", 9, 1.49),+ ...cluster("speaker_03", 2, 0.72),+ ];+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_01", "speaker_06"]);+});++// The same call clustered at 0.65: the larger phantom held 5.6 % of the speech+// (59.6 s of 1058.4 s) in segments averaging 1.92 s.+test("a short-segment phantom under 10% of the speech is dropped", () => {+ const segments = [+ ...cluster("speaker_0", 94, 5.77),+ ...cluster("speaker_1", 83, 5.44),+ ...cluster("speaker_2", 31, 1.92),+ ...cluster("speaker_3", 3, 1.7),+ ];+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0", "speaker_1"]);+});++test("a minor third speaker with real turns survives", () => {+ const segments = [+ ...cluster("speaker_0", 80, 6),+ ...cluster("speaker_1", 80, 6),+ ...cluster("speaker_2", 7, 6),+ ...cluster("speaker_3", 20, 1.5),+ ];+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), [+ "speaker_0",+ "speaker_1",+ "speaker_2",+ ]);+});++test("a short-turn speaker with a real share of the speech survives", () => {+ const segments = [+ ...cluster("speaker_0", 100, 2),+ ...cluster("speaker_1", 100, 2),+ ...cluster("speaker_2", 40, 2),+ ];+ assert.equal(speakersOf(dropNegligibleClusters(segments)).length, 3);+});++test("the second-largest cluster survives even when it looks like a phantom", () => {+ const segments = [+ ...cluster("speaker_0", 200, 6),+ ...cluster("speaker_1", 20, 1.5),+ ...cluster("speaker_2", 10, 1.5),+ ];+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0", "speaker_1"]);+});++// Forcing two clusters on #2021's call put both real speakers in one cluster+// and kept a phantom for the rest.+test("one cluster holding nearly all the speech is a collapsed run", () => {+ const segments = [...cluster("speaker_01", 165, 6.06), ...cluster("speaker_00", 35, 1.72)];+ assert.equal(isCollapsedDiarization(segments), true);+});++test("a balanced run or a single cluster is not a collapsed run", () => {+ const balanced = [...cluster("speaker_06", 79, 6.46), ...cluster("speaker_01", 85, 5.44)];+ assert.equal(isCollapsedDiarization(balanced), false);+ assert.equal(isCollapsedDiarization(cluster("speaker_0", 50, 6)), false);+ assert.equal(isCollapsedDiarization([]), false);+});++test("a dominant speaker beside someone with real turns is not a collapsed run", () => {+ const lecture = [...cluster("speaker_0", 550, 6), ...cluster("speaker_1", 5, 6)];+ assert.equal(isCollapsedDiarization(lecture), false);+});+test("dropping never empties the result or touches a clean input", () => {const allTiny = [{ start: 0, end: 0.3, speaker: "speaker_0" },@@ -98,3 +195,169 @@ test("dropping never empties the result or touches a clean input", () => {test("minimum speaking time constant is sane", () => {assert.ok(MIN_CLUSTER_TOTAL_SECONDS >= 0.5 && MIN_CLUSTER_TOTAL_SECONDS <= 5);});++// sherpa numbers clusters by first appearance, and on #2021's call the first+// cluster (speaker_00) was a phantom. Protection must follow size, not order.+test("the protected clusters are the largest, whatever order sherpa lists them in", () => {+ const segments = [+ ...cluster("speaker_00", 23, 1.72),+ ...cluster("speaker_01", 85, 5.44),+ ...cluster("speaker_02", 20, 1.57),+ ...cluster("speaker_06", 79, 6.46),+ ];+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_01", "speaker_06"]);+});++test("the 1 s floor applies to the protected second-largest cluster", () => {+ const segments = [...cluster("speaker_0", 5, 6), ...cluster("speaker_1", 2, 0.4)];+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0"]);+});++test("a short-segment cluster at exactly 10% of the speech is kept", () => {+ const segments = [+ ...cluster("speaker_0", 45, 10),+ ...cluster("speaker_1", 45, 10),+ ...cluster("speaker_2", 50, 2),+ ];+ assert.equal(speakersOf(dropNegligibleClusters(segments)).length, 3);+});++test("a small cluster is kept from a 3 s mean segment up and dropped below it", () => {+ const withMean = (seconds, count) => [+ ...cluster("speaker_0", 47, 10),+ ...cluster("speaker_1", 47, 10),+ ...cluster("speaker_2", count, seconds),+ ];+ assert.equal(speakersOf(dropNegligibleClusters(withMean(3, 20))).length, 3);+ assert.equal(speakersOf(dropNegligibleClusters(withMean(4, 15))).length, 3);+ assert.equal(speakersOf(dropNegligibleClusters(withMean(2.5, 20))).length, 2);+});++// Documented cost of the phantom rule: by duration alone, a real third+// participant with under 10 % of the speech in short turns looks like a+// phantom. Flip this deliberately if the rule learns to use voices.+test("characterization: a quiet third participant in short turns is dropped", () => {+ const segments = [+ ...cluster("speaker_0", 60, 7.5),+ ...cluster("speaker_1", 60, 7.5),+ ...cluster("speaker_2", 34, 2.8),+ ];+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0", "speaker_1"]);+});++test("a collapsed run is found whatever order sherpa lists its clusters in", () => {+ const segments = [...cluster("speaker_00", 35, 1.72), ...cluster("speaker_01", 165, 6.06)];+ assert.equal(isCollapsedDiarization(segments), true);+});++test("a run is collapsed only above 93% for the largest cluster", () => {+ const withTop = (seconds) => [+ ...cluster("speaker_0", 1, seconds),+ ...cluster("speaker_1", 35, 2),+ ];+ assert.equal(isCollapsedDiarization(withTop(930)), false);+ assert.equal(isCollapsedDiarization(withTop(931)), true);+});++// Short turns alone don't make a collapse: someone holding 15 % of the speech+// in quick replies is a real second speaker.+test("a lopsided conversation in short turns on both sides is not collapsed", () => {+ const segments = [...cluster("speaker_0", 425, 2), ...cluster("speaker_1", 75, 2)];+ assert.equal(isCollapsedDiarization(segments), false);+});++test("a run is not collapsed while any other cluster averages 2.2 s segments or more", () => {+ const beside = (seconds) => [+ ...cluster("speaker_0", 100, 10),+ ...cluster("speaker_1", 1, seconds),+ ];+ assert.equal(isCollapsedDiarization(beside(2.2)), false);+ assert.equal(isCollapsedDiarization(beside(2.19)), true);++ const oneRealBesidePhantom = [+ ...cluster("speaker_0", 91, 10),+ ...cluster("speaker_1", 10, 5),+ ...cluster("speaker_2", 20, 2),+ ];+ assert.equal(isCollapsedDiarization(oneRealBesidePhantom), false);+});++// The 1 s floor exists for these blips; a monologue must keep its label.+test("a single speaker beside a sub-second blip is not collapsed", () => {+ const monologue = [...cluster("speaker_0", 100, 6), ...cluster("speaker_1", 1, 0.45)];+ assert.equal(isCollapsedDiarization(monologue), false);++ const blips = [+ ...cluster("speaker_0", 100, 6),+ ...cluster("speaker_1", 1, 0.5),+ ...cluster("speaker_2", 2, 0.4),+ ];+ assert.equal(isCollapsedDiarization(blips), false);+});++test("interviews, podcasts and lectures dominated by one voice keep their labels", () => {+ const oralHistory = [...cluster("speaker_0", 40, 60), ...cluster("speaker_1", 40, 2.5)];+ const podcast = [...cluster("speaker_0", 60, 20), ...cluster("speaker_1", 40, 2.9)];+ const lecture = [...cluster("speaker_0", 500, 6), ...cluster("speaker_1", 6, 2.8)];+ for (const run of [oralHistory, podcast, lecture]) {+ assert.equal(isCollapsedDiarization(run), false);+ }+});++test("characterization: a listener who only backchannels beside a dominant speaker reads as collapsed", () => {+ const segments = [...cluster("speaker_0", 100, 10), ...cluster("speaker_1", 30, 1.8)];+ assert.equal(isCollapsedDiarization(segments), true);+});++const dot = (a, b) => a.reduce((sum, value, i) => sum + value * b[i], 0);++test("an extra cluster is relabelled to the kept speaker it sounds like", () => {+ const segments = [+ ...cluster("speaker_0", 10, 10),+ ...cluster("speaker_1", 8, 10),+ ...cluster("speaker_2", 6, 10),+ ];+ const centroids = new Map([+ ["speaker_0", [1, 0]],+ ["speaker_1", [0, 1]],+ ["speaker_2", [0.1, 0.9]],+ ]);+ const capped = capSpeakerClustersByVoice(segments, 2, centroids, dot);+ assert.deepEqual(speakersOf(capped), ["speaker_0", "speaker_1"]);+ assert.equal(capped.filter((s) => s.speaker === "speaker_1").length, 14);+ assert.deepEqual(+ capped.slice(0, 18).map((s) => s.speaker),+ segments.slice(0, 18).map((s) => s.speaker)+ );+});++test("an extra cluster without a usable voice folds into the largest", () => {+ const segments = [+ ...cluster("speaker_0", 10, 10),+ ...cluster("speaker_1", 8, 10),+ ...cluster("speaker_2", 6, 10),+ ];+ const noExtraVoice = new Map([+ ["speaker_0", [1, 0]],+ ["speaker_1", [0, 1]],+ ]);+ const capped = capSpeakerClustersByVoice(segments, 2, noExtraVoice, dot);+ assert.equal(capped.filter((s) => s.speaker === "speaker_0").length, 16);++ // A kept speaker without a voice can't be matched, so the extra goes to the+ // closest kept speaker that has one.+ const onlySecondVoice = new Map([+ ["speaker_1", [0, 1]],+ ["speaker_2", [1, 0]],+ ]);+ const matched = capSpeakerClustersByVoice(segments, 2, onlySecondVoice, dot);+ assert.equal(matched.filter((s) => s.speaker === "speaker_1").length, 14);+});++test("the voice cap leaves a run within the cap untouched and can fold everything into one", () => {+ const segments = [...cluster("speaker_0", 10, 10), ...cluster("speaker_1", 8, 10)];+ assert.equal(capSpeakerClustersByVoice(segments, 2, new Map(), dot), segments);+ assert.deepEqual(speakersOf(capSpeakerClustersByVoice(segments, 1, new Map(), dot)), [+ "speaker_0",+ ]);+});diff --git a/test/helpers/diarizeAudioFileHandler.test.js b/test/helpers/diarizeAudioFileHandler.test.jsnew file mode 100644index 000000000..3959015d2--- /dev/null+++ b/test/helpers/diarizeAudioFileHandler.test.js@@ -0,0 +1,244 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const Module = require("node:module");+const fs = require("node:fs");+const os = require("node:os");+const path = require("node:path");++const handlersModulePath = require.resolve("../../src/helpers/ipcHandlers");+const originalLoad = Module._load;+const handlers = new Map();++const electronStub = {+ app: {+ getPath: () => os.tmpdir(),+ getName: () => "test",+ getVersion: () => "0.0.0",+ isPackaged: false,+ on: () => {},+ requestSingleInstanceLock: () => true,+ },+ ipcMain: {+ handle: (channel, fn) => handlers.set(channel, fn),+ on: () => {},+ removeHandler: () => {},+ },+ net: {},+ BrowserWindow: class BrowserWindow {+ static getAllWindows() {+ return [];+ }+ static fromWebContents() {+ return null;+ }+ },+ shell: {},+ dialog: {},+ screen: { getPrimaryDisplay: () => ({ workAreaSize: { width: 0, height: 0 } }) },+ systemPreferences: { getMediaAccessStatus: () => "granted" },+ session: { fromPartition: () => ({}) },+ clipboard: {},+ nativeImage: {},+ globalShortcut: {},+ utilityProcess: {},+ MessageChannelMain: class {},+};++// Voices are looked up by cluster id; tests set the centroids (or make the+// lookup fail) per case.+let centroidCalls = [];+let clusterCentroids = async () => new Map();+const speakerEmbeddingsStub = {+ extractClusterCentroids: (...args) => {+ centroidCalls.push(args);+ return clusterCentroids(...args);+ },+ cosineSimilarity: (a, b) => a.reduce((sum, value, i) => sum + value * b[i], 0),+};++// ffmpeg is not run: the handler only needs a WAV whose size gives a duration.+Module._load = function loadWithMocks(request, parent, isMain) {+ if (request === "electron") return electronStub;+ if (parent?.filename === handlersModulePath && request === "./ffmpegUtils") {+ return {+ ...originalLoad.call(this, request, parent, isMain),+ convertToWav: async (_input, output) => fs.writeFileSync(output, Buffer.alloc(32000)),+ };+ }+ if (parent?.filename === handlersModulePath && request === "./speakerEmbeddings") {+ return speakerEmbeddingsStub;+ }+ return originalLoad.call(this, request, parent, isMain);+};++function anything() {+ return new Proxy(function () {}, {+ get: (t, prop) => {+ if (prop === Symbol.toPrimitive || prop === "toString") return () => "";+ if (prop === "then") return undefined;+ return anything();+ },+ apply: () => anything(),+ });+}++let diarizeCalls = [];+let diarizerOutput = [];+let audioPath;+let handler;++test.before(() => {+ delete require.cache[handlersModulePath];+ const IPCHandlers = require(handlersModulePath);+ const Ctor = IPCHandlers.default || IPCHandlers;+ const target = {+ _uploadCancelRegistry: { register: () => ({ signal: undefined, release: () => {} }) },+ diarizationManager: {+ isModelDownloaded: () => true,+ diarize: async (_wavPath, options) => {+ diarizeCalls.push(options);+ return diarizerOutput;+ },+ },+ };+ Ctor.prototype.setupHandlers.call(+ new Proxy(target, { get: (t, prop) => (prop in t ? t[prop] : anything()) })+ );+ handler = handlers.get("diarize-audio-file");+ assert.ok(handler, "diarize-audio-file must be registered");++ audioPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "ow-diarize-test-")), "call.mp3");+ fs.writeFileSync(audioPath, "");+ Ctor.prototype.approveAudioPath(audioPath);+});++test.beforeEach(() => {+ diarizeCalls = [];+ centroidCalls = [];+ clusterCentroids = async () => new Map();+});++test.after(() => {+ Module._load = originalLoad;+ fs.rmSync(path.dirname(audioPath), { recursive: true, force: true });+});++function cluster(speaker, count, seconds) {+ return Array.from({ length: count }, (_, i) => ({+ start: i * 20,+ end: i * 20 + seconds,+ speaker,+ }));+}++const speakersOf = (segments) => [...new Set(segments.map((s) => s.speaker))].sort();++// Forcing sherpa to 2 clusters on #2021's two-person call split short+// utterances from long ones, merging both real speakers and keeping a phantom.+// The requested count must never reach sherpa.+test("a requested speaker count still clusters automatically", async () => {+ diarizerOutput = [+ ...cluster("speaker_06", 79, 6.46),+ ...cluster("speaker_01", 85, 5.44),+ ...cluster("speaker_00", 23, 1.72),+ ];++ const result = await handler({}, audioPath, { numSpeakers: 2 });++ assert.equal(result.success, true);+ assert.equal(diarizeCalls.length, 1);+ // diarize() maps a missing count to --clustering.num-clusters=-1 (auto); its+ // default doesn't replace null, which would reach sherpa as "null".+ assert.equal(Object.hasOwn(diarizeCalls[0], "numSpeakers"), false);+ assert.deepEqual(speakersOf(result.segments), ["speaker_01", "speaker_06"]);+ assert.equal(centroidCalls.length, 0, "voices are only read when clusters exceed the cap");+});++test("a collapsed run returns no segments, so the plain transcript is kept", async () => {+ diarizerOutput = [...cluster("speaker_01", 165, 6.06), ...cluster("speaker_00", 35, 1.72)];++ const result = await handler({}, audioPath, {});++ assert.equal(result.success, true);+ assert.deepEqual(result.segments, []);+ assert.equal(result.durationSeconds, 1);+});++test("a requested speaker count caps the clusters automatic clustering finds", async () => {+ diarizerOutput = [+ ...cluster("speaker_0", 80, 6),+ ...cluster("speaker_1", 60, 6),+ ...cluster("speaker_2", 40, 6),+ ];++ const result = await handler({}, audioPath, { numSpeakers: 2 });++ assert.deepEqual(speakersOf(result.segments), ["speaker_0", "speaker_1"]);+});++test("an extra cluster goes to the requested speaker whose voice it matches", async () => {+ diarizerOutput = [+ ...cluster("speaker_0", 80, 6),+ ...cluster("speaker_1", 60, 6),+ ...cluster("speaker_2", 40, 6),+ ];+ clusterCentroids = async () =>+ new Map([+ ["speaker_0", [1, 0]],+ ["speaker_1", [0, 1]],+ ["speaker_2", [0.2, 0.8]],+ ]);++ const result = await handler({}, audioPath, { numSpeakers: 2 });++ assert.equal(centroidCalls.length, 1);+ assert.equal(result.segments.filter((s) => s.speaker === "speaker_1").length, 100);+ assert.equal(result.segments.filter((s) => s.speaker === "speaker_0").length, 80);+});++test("extra clusters fold into the largest when voices can't be read", async () => {+ diarizerOutput = [+ ...cluster("speaker_0", 80, 6),+ ...cluster("speaker_1", 60, 6),+ ...cluster("speaker_2", 40, 6),+ ];+ clusterCentroids = async () => {+ throw new Error("Speaker embedding model not found");+ };++ const result = await handler({}, audioPath, { numSpeakers: 2 });++ assert.equal(result.success, true);+ assert.equal(result.segments.filter((s) => s.speaker === "speaker_0").length, 120);+});++// Dropping the 3 % phantom first would lift the interviewee from 88 % to+// 90.7 % of the speech and discard a correctly labelled interview.+test("a solo recording with a stray blip keeps its one speaker", async () => {+ diarizerOutput = [...cluster("speaker_0", 100, 6), ...cluster("speaker_1", 1, 0.45)];++ const result = await handler({}, audioPath, {});++ assert.deepEqual(speakersOf(result.segments), ["speaker_0"]);+});++test("an interview with short questions keeps its labels", async () => {+ diarizerOutput = [+ ...cluster("speaker_0", 88, 10),+ ...cluster("speaker_1", 36, 2.5),+ ...cluster("speaker_2", 20, 1.5),+ ];++ const result = await handler({}, audioPath, {});++ assert.deepEqual(speakersOf(result.segments), ["speaker_0", "speaker_1"]);+});++test("a request for one speaker labels one speaker instead of discarding the run", async () => {+ diarizerOutput = [...cluster("speaker_01", 165, 6.06), ...cluster("speaker_00", 35, 1.72)];++ const result = await handler({}, audioPath, { numSpeakers: 1 });++ assert.equal(result.segments.length, 200);+ assert.deepEqual(speakersOf(result.segments), ["speaker_01"]);+});diff --git a/test/helpers/speakerEmbeddingsClusterCentroids.test.js b/test/helpers/speakerEmbeddingsClusterCentroids.test.jsnew file mode 100644index 000000000..72f16dfb7--- /dev/null+++ b/test/helpers/speakerEmbeddingsClusterCentroids.test.js@@ -0,0 +1,183 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const Module = require("node:module");+const fs = require("node:fs");+const os = require("node:os");+const path = require("node:path");++const speakerModulePath = require.resolve("../../src/helpers/speakerEmbeddings");+const originalLoad = Module._load;+Module._load = function loadWithMocks(request, parent, isMain) {+ if (request === "electron") return { app: { getPath: () => os.tmpdir(), isPackaged: false } };+ if (parent?.filename === speakerModulePath && request === "./onnxWorkerClient") return {};+ return originalLoad.call(this, request, parent, isMain);+};+const speakerEmbeddings = require(speakerModulePath);+Module._load = originalLoad;++const SAMPLE_RATE = 16000;++// A 16 kHz mono PCM16 WAV whose level changes every 10 s, so an embedding+// stub can tell which region of the file it was handed.+function writeWav(levels) {+ const samplesPerRegion = SAMPLE_RATE * 10;+ const data = Buffer.alloc(levels.length * samplesPerRegion * 2);+ levels.forEach((level, region) => {+ for (let i = 0; i < samplesPerRegion; i++) {+ data.writeInt16LE(Math.round(level * 32768), (region * samplesPerRegion + i) * 2);+ }+ });+ const header = Buffer.alloc(44);+ header.write("RIFF", 0);+ header.writeUInt32LE(36 + data.length, 4);+ header.write("WAVE", 8);+ header.write("fmt ", 12);+ header.writeUInt32LE(16, 16);+ header.writeUInt16LE(1, 20);+ header.writeUInt16LE(1, 22);+ header.writeUInt32LE(SAMPLE_RATE, 24);+ header.writeUInt32LE(SAMPLE_RATE * 2, 28);+ header.writeUInt16LE(2, 32);+ header.writeUInt16LE(16, 34);+ header.write("data", 36);+ header.writeUInt32LE(data.length, 40);++ const wavPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "ow-centroids-")), "a.wav");+ fs.writeFileSync(wavPath, Buffer.concat([header, data]));+ return wavPath;+}++const segments = [+ { start: 0, end: 4, speaker: "speaker_a" },+ { start: 4, end: 6.5, speaker: "speaker_a" },+ { start: 6.5, end: 8.5, speaker: "speaker_a" },+ { start: 8.4, end: 10, speaker: "speaker_a" },+ { start: 10, end: 12, speaker: "speaker_b" },+ { start: 12, end: 13, speaker: "speaker_c" },+];++let extracted;+let wavPath;++test.before(() => {+ wavPath = writeWav([0.25, 0.5]);+ speakerEmbeddings._extractEmbeddingFromSamples = async (samples) => {+ extracted.push(samples);+ return new Float32Array(512).fill(samples[0]);+ };+});++test.beforeEach(() => {+ extracted = [];+});++test.after(() => {+ fs.rmSync(path.dirname(wavPath), { recursive: true, force: true });+});++test("each cluster's voice comes from its three longest segments of 1.5 s or more", async () => {+ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments);++ assert.deepEqual([...centroids.keys()], ["speaker_a", "speaker_b"]);+ assert.deepEqual(+ extracted.map((samples) => samples.length / SAMPLE_RATE),+ [4, 2.5, 2, 2]+ );+ assert.ok(Math.abs(centroids.get("speaker_a")[0] - 0.25) < 1e-4);+ assert.ok(Math.abs(centroids.get("speaker_b")[0] - 0.5) < 1e-4);+});++test("only the audio each voice uses is read, never the whole WAV", async () => {+ const originalReadFile = fs.readFileSync;+ const originalRead = fs.readSync;+ let wholeFileReads = 0;+ const reads = [];+ fs.readFileSync = (file, ...rest) => {+ if (file === wavPath) wholeFileReads += 1;+ return originalReadFile(file, ...rest);+ };+ fs.readSync = (fd, buffer, offset, length, position) => {+ reads.push(length);+ return originalRead(fd, buffer, offset, length, position);+ };+ const originalOpen = fs.openSync;+ const originalClose = fs.closeSync;+ let openFiles = 0;+ fs.openSync = (...args) => {+ openFiles += 1;+ return originalOpen(...args);+ };+ fs.closeSync = (fd) => {+ openFiles -= 1;+ return originalClose(fd);+ };+ try {+ await speakerEmbeddings.extractClusterCentroids(wavPath, segments);+ } finally {+ fs.readFileSync = originalReadFile;+ fs.readSync = originalRead;+ fs.openSync = originalOpen;+ fs.closeSync = originalClose;+ }+ assert.equal(wholeFileReads, 0);+ assert.equal(openFiles, 0);+ // The header, then one window per embedded segment.+ assert.deepEqual(+ reads.slice(1).map((bytes) => bytes / (SAMPLE_RATE * 2)),+ [4, 2.5, 2, 2]+ );+});++test("a segment longer than 8 s is embedded from its last 8 s", async () => {+ await speakerEmbeddings.extractClusterCentroids(wavPath, [+ { start: 0, end: 12, speaker: "speaker_a" },+ ]);++ const [samples] = extracted;+ assert.equal(samples.length / SAMPLE_RATE, 8);+ assert.ok(Math.abs(samples[0] - 0.25) < 1e-4);+ assert.ok(Math.abs(samples[samples.length - 1] - 0.5) < 1e-4);+});++test("a cluster whose voice fails leaves the other clusters' voices intact", async () => {+ const workingExtract = speakerEmbeddings._extractEmbeddingFromSamples;+ speakerEmbeddings._extractEmbeddingFromSamples = async (samples) => {+ if (Math.abs(samples[0] - 0.25) < 1e-4) throw new Error("onnx worker request timeout");+ return workingExtract(samples);+ };+ try {+ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments);+ assert.deepEqual([...centroids.keys()], ["speaker_b"]);+ } finally {+ speakerEmbeddings._extractEmbeddingFromSamples = workingExtract;+ }+});++test("a cancel between clusters stops before the next cluster", async () => {+ const controller = new AbortController();+ const workingExtract = speakerEmbeddings._extractEmbeddingFromSamples;+ speakerEmbeddings._extractEmbeddingFromSamples = async (samples) => {+ controller.abort();+ return workingExtract(samples);+ };+ try {+ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments, {+ signal: controller.signal,+ });+ assert.deepEqual([...centroids.keys()], ["speaker_a"]);+ } finally {+ speakerEmbeddings._extractEmbeddingFromSamples = workingExtract;+ }+});++test("a cancelled upload stops before extracting voices", async () => {+ const controller = new AbortController();+ controller.abort();++ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments, {+ signal: controller.signal,+ });++ assert.equal(centroids.size, 0);+ assert.equal(extracted.length, 0);+});diff --git a/test/services/fileTranscriptionDiarizationWarning.test.js b/test/services/fileTranscriptionDiarizationWarning.test.jsindex ff6dd3aa2..348b32432 100644--- a/test/services/fileTranscriptionDiarizationWarning.test.js+++ b/test/services/fileTranscriptionDiarizationWarning.test.js@@ -75,6 +75,32 @@ test("keeps the transcript and warns when speaker labels cannot be merged", asynassert.equal(result.diarizationWarning, true);});+// The diarize handler returns no segments when it discards a collapsed run.+test("warns and keeps the measured duration when diarization returns no segments", async (t) => {+ const { window, transcribeFileWithSpeakers } = await loadFileTranscription(t);+ let mergeCalls = 0;+ window.electronAPI.diarizeAudioFile = async () => ({+ success: true,+ segments: [],+ durationSeconds: 1188,+ });+ window.electronAPI.mergeSpeakerText = async () => {+ mergeCalls += 1;+ return { success: true, text: "Speaker 1: Plain transcript" };+ };++ const result = await transcribeFileWithSpeakers("/tmp/audio.wav", localConfig(), {+ enabled: true,+ localModelsReady: true,+ numSpeakers: null,+ });++ assert.equal(mergeCalls, 0);+ assert.equal(result.text, "Plain transcript");+ assert.equal(result.diarizationWarning, true);+ assert.equal(result.durationSeconds, 1188);+});+test("does not warn when speaker labels are merged", async (t) => {const { window, transcribeFileWithSpeakers } = await loadFileTranscription(t);window.electronAPI.diarizeAudioFile = async () => ({diff --git a/test/services/uploadDiarizationPersistence.test.js b/test/services/uploadDiarizationPersistence.test.jsindex 213c36451..21dbccb8e 100644--- a/test/services/uploadDiarizationPersistence.test.js+++ b/test/services/uploadDiarizationPersistence.test.js@@ -3,10 +3,9 @@ const assert = require("node:assert/strict");const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");// Regression for the upload/URL-ingest path dropping diarization metadata: the-// batch queue ran the diarizer but persisted none of diarization_enabled,-// expected_speaker_count, or audio_duration_seconds, so an uploaded note that-// WAS diarized presented to the app as though it was not (the meeting path-// persists all of them).+// batch queue ran the diarizer but persisted neither diarization_enabled nor+// audio_duration_seconds, so an uploaded note that WAS diarized presented to+// the app as though it was not (the meeting path persists both).const transcription = {useLocalWhisper: true,@@ -105,7 +104,7 @@ test("a diarized file upload persists the diarization metadata", async (t) => {// The diarizer runs under the same cancellable requestId as the// transcription, so one cancel-upload-transcription aborts both.assert.equal(typeof calls.diarize[0].requestId, "string");- assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1, expected_speaker_count: 2 }]]);+ assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1 }]]);});test("a diarized URL ingest persists the diarization metadata", async (t) => {@@ -141,7 +140,7 @@ test("a diarized URL ingest persists the diarization metadata", async (t) => {assert.equal(sourceFile, "Board Meeting Recording");assert.equal(audioDuration, 4359.87);- assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1, expected_speaker_count: 2 }]]);+ assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1 }]]);assert.deepEqual(calls.deletedTempFiles, ["/tmp/ow-url-download.m4a"]);});diff --git a/test/services/uploadNotes.test.js b/test/services/uploadNotes.test.jsindex 39f1faeef..11279df1a 100644--- a/test/services/uploadNotes.test.js+++ b/test/services/uploadNotes.test.js@@ -4,7 +4,7 @@ const assert = require("node:assert/strict");const load = () => import("../../src/services/uploadNotes.ts");const { MAX_SPEAKER_COUNT } = require("../../src/constants/speakerDetection.json");-test("maps the diarizer invocation onto the note columns", async () => {+test("maps the diarization settings onto the note columns", async () => {const { buildUploadNoteMetadata } = await load();const { audioDurationSeconds, noteUpdates } = buildUploadNoteMetadata(@@ -13,30 +13,29 @@ test("maps the diarizer invocation onto the note columns", async () => {);assert.equal(audioDurationSeconds, 4359.87);- assert.deepEqual(noteUpdates, { diarization_enabled: 1, expected_speaker_count: 2 });+ assert.deepEqual(noteUpdates, { diarization_enabled: 1 });});-test("auto speaker detection stores no explicit count", async () => {+// The upload caps detected speakers at the entered count, but recording into+// the note later forces expected_speaker_count as the exact count.+test("the speaker count entered for an upload is never stored on the note", async () => {const { buildUploadNoteMetadata } = await load();- const { noteUpdates } = buildUploadNoteMetadata(- { enabled: true, localModelsReady: true, numSpeakers: null },- 60- );-- // isExplicitSpeakerCount treats any positive integer as the user's own- // choice, so auto detection must land as null, never a default.- assert.equal(noteUpdates.expected_speaker_count, null);- assert.equal(noteUpdates.diarization_enabled, 1);+ for (const numSpeakers of [null, 2, MAX_SPEAKER_COUNT]) {+ const { noteUpdates } = buildUploadNoteMetadata(+ { enabled: true, localModelsReady: true, numSpeakers },+ 60+ );+ assert.equal(Object.hasOwn(noteUpdates, "expected_speaker_count"), false);+ assert.equal(noteUpdates.diarization_enabled, 1);+ }});test("disabled diarization writes no columns at all", async () => {const { buildUploadNoteMetadata } = await load();// A null diarization_enabled defers to the global speaker setting when the- // user records into the note later (a 0 would force it off), and writing no- // count keeps the numSpeakers value lingering in localStorage while its- // input is hidden from being stamped onto the note as an explicit choice.+ // user records into the note later (a 0 would force it off).const { audioDurationSeconds, noteUpdates } = buildUploadNoteMetadata({ enabled: false, localModelsReady: false, numSpeakers: 3 },120@@ -46,22 +45,6 @@ test("disabled diarization writes no columns at all", async () => {assert.equal(audioDurationSeconds, 120);});-test("speaker counts reuse the stored-count clamp", async () => {- const { buildUploadNoteMetadata } = await load();-- const above = buildUploadNoteMetadata(- { enabled: true, localModelsReady: true, numSpeakers: MAX_SPEAKER_COUNT + 5 },- null- );- assert.equal(above.noteUpdates.expected_speaker_count, MAX_SPEAKER_COUNT);-- const unusable = buildUploadNoteMetadata(- { enabled: true, localModelsReady: true, numSpeakers: 0 },- null- );- assert.equal(unusable.noteUpdates.expected_speaker_count, null);-});-test("unusable durations degrade to null", async () => {const { buildUploadNoteMetadata } = await load();const diarization = { enabled: true, localModelsReady: true, numSpeakers: 2 };@@ -183,7 +166,6 @@ test("segments and diarization metadata share one noteUpdates write", async () =assert.deepEqual(noteUpdates, {diarization_enabled: 1,- expected_speaker_count: 2,transcript: JSON.stringify([{ text: "Hi.", timestamp: anchorMs + 1000, speakerName: "Speaker 1" },]),@@ -200,21 +182,3 @@ test("upload titles fall back to the transcript, then the file name", async () =);assert.equal(uploadTitleFallback(" ", "board-meeting.m4a"), "board-meeting");});--// uploadNotes.ts carries a renderer twin of the main-process-// normalizeStoredSpeakerCount (CJS, unloadable from renderer source). Hold the-// two implementations to identical outputs so they cannot drift apart.-test("speaker-count normalization matches the main-process implementation", async () => {- const { buildUploadNoteMetadata } = await load();- const { normalizeStoredSpeakerCount } = require("../../src/helpers/speakerCount");-- const inputs = [1, 2, 3, MAX_SPEAKER_COUNT, MAX_SPEAKER_COUNT + 1, MAX_SPEAKER_COUNT + 5];- for (const value of [...inputs, 0, -1, 1.5, NaN, Infinity, -Infinity, null, undefined, "", {}]) {- assert.equal(- buildUploadNoteMetadata({ enabled: true, numSpeakers: value }).noteUpdates- .expected_speaker_count,- normalizeStoredSpeakerCount(value),- `diverged from normalizeStoredSpeakerCount for ${JSON.stringify(value)}`- );- }-});