Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(diarization): stop phantom speakers in uploaded recordings (#2403)

void13afd03e54d6
diff --git a/src/components/notes/UploadAudioView.tsx b/src/components/notes/UploadAudioView.tsx
index b2a8265fe..e801133d4 100644
--- a/src/components/notes/UploadAudioView.tsx
+++ b/src/components/notes/UploadAudioView.tsx
@@ -189,9 +189,11 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
const [diarizationEnabled, setDiarizationEnabled] = useState(
() => localStorage.getItem("uploadDiarizationEnabled") === "true"
);
- const [diarizationNumSpeakers, setDiarizationNumSpeakers] = useState<string>(
- () => localStorage.getItem("uploadDiarizationNumSpeakers") || ""
- );
+ // Earlier builds accepted decimals; a saved one would read as Auto.
+ const [diarizationNumSpeakers, setDiarizationNumSpeakers] = useState<string>(() => {
+ const saved = localStorage.getItem("uploadDiarizationNumSpeakers") || "";
+ return /^\d+$/.test(saved) ? saved : "";
+ });
const [diarizationModelsReady, setDiarizationModelsReady] = useState<boolean | null>(null);
const [diarizationDownloading, setDiarizationDownloading] = useState(false);
@@ -1319,15 +1321,21 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
type="number"
min="2"
max={MAX_SPEAKER_COUNT}
+ step="1"
+ inputMode="numeric"
value={diarizationNumSpeakers}
+ onKeyDown={(e) => {
+ if ([".", ",", "e", "E", "+", "-"].includes(e.key)) e.preventDefault();
+ }}
onChange={(e) => {
const raw = e.target.value;
if (raw === "") {
setDiarizationNumSpeakers("");
return;
}
- const n = Math.max(2, Math.min(MAX_SPEAKER_COUNT, Number(raw)));
- setDiarizationNumSpeakers(String(isNaN(n) ? "" : n));
+ const n = Number(raw);
+ if (!Number.isInteger(n)) return;
+ setDiarizationNumSpeakers(String(Math.max(2, Math.min(MAX_SPEAKER_COUNT, n))));
}}
placeholder={t("notes.upload.numSpeakersPlaceholder")}
className="h-9 w-32 shrink-0 rounded-xl! px-3 text-sm placeholder:text-foreground/45 [appearance:textfield] [&::-webkit-outer-spin-button]:appearance-none [&::-webkit-inner-spin-button]:appearance-none"
diff --git a/src/helpers/diarizationPolicy.js b/src/helpers/diarizationPolicy.js
index 39c47c2c1..1a587a7ce 100644
--- a/src/helpers/diarizationPolicy.js
+++ b/src/helpers/diarizationPolicy.js
@@ -6,8 +6,10 @@
// speaker's embeddings drift wider than that and agglomerative clustering
// stops merging early — a 73-minute voice memo produced 46 "speakers". Ramp
// the threshold with duration instead of pretending one constant fits both.
+// The top stays below 0.72: from there up, a two-person Spanish call merged
+// both speakers into one cluster.
const DEFAULT_CLUSTER_THRESHOLD = 0.55;
-const LONG_AUDIO_CLUSTER_THRESHOLD = 0.8;
+const LONG_AUDIO_CLUSTER_THRESHOLD = 0.7;
const THRESHOLD_RAMP_START_SECONDS = 15 * 60;
const THRESHOLD_RAMP_END_SECONDS = 60 * 60;
@@ -16,6 +18,31 @@ const THRESHOLD_RAMP_END_SECONDS = 60 * 60;
// at least one sentence, so each blip becomes a phantom [Speaker N].
const MIN_CLUSTER_TOTAL_SECONDS = 1;
+// Longer recordings grow phantoms well past that floor: backchannels and turn
+// starts whose 1–3 s embeddings land near neither speaker's centroid (a
+// 20-minute two-person call produced four, holding 8 % of the speech). A
+// phantom is both a sliver of the speech and made of short segments; a real
+// minor speaker holds longer turns. Known cost: a real third participant with
+// under 10 % of the speech in turns averaging under 3 s looks the same and is
+// dropped too. Only their voice tells them apart, which this rule doesn't use.
+const PHANTOM_MAX_SHARE = 0.1;
+const PHANTOM_MAX_MEAN_SEGMENT_SECONDS = 3;
+// Merging two real speakers is the worse failure, so the two largest clusters
+// are never treated as phantoms.
+const PROTECTED_CLUSTER_COUNT = 2;
+
+// When clustering merges two real speakers, one cluster ends up with nearly
+// all the speech beside phantoms of backchannels and turn starts: on every
+// collapsed run of #2021's call the top held 94–96 % and each phantom
+// averaged 1.7–2 s segments. Someone who asks questions, however short,
+// averages longer segments, so an interview or a lecture with audience
+// questions keeps its labels. Known cost: a listener who only backchannels
+// beside a speaker holding over 93 % looks the same, and that run loses its
+// labels. Clusters under the 1 s floor are noise and don't count, so a
+// monologue with a stray blip isn't a collapse.
+const COLLAPSED_TOP_SHARE = 0.93;
+const COLLAPSED_MAX_MEAN_SEGMENT_SECONDS = 2.2;
+
function clusterThresholdForDuration(durationSeconds) {
if (!Number.isFinite(durationSeconds) || durationSeconds <= THRESHOLD_RAMP_START_SECONDS) {
return DEFAULT_CLUSTER_THRESHOLD;
@@ -38,19 +65,85 @@ function resolveClusterThreshold(durationSeconds, requestedThreshold) {
return Math.min(1, Math.max(0, parsedThreshold));
}
-function dropNegligibleClusters(segments, minTotalSeconds = MIN_CLUSTER_TOTAL_SECONDS) {
- if (!segments?.length) return segments;
- const totals = new Map();
+function summarizeClusters(segments) {
+ const clusters = new Map();
+ let totalSpeech = 0;
for (const s of segments) {
- totals.set(s.speaker, (totals.get(s.speaker) || 0) + (s.end - s.start));
+ const cluster = clusters.get(s.speaker) || { total: 0, count: 0 };
+ cluster.total += s.end - s.start;
+ cluster.count += 1;
+ clusters.set(s.speaker, cluster);
+ totalSpeech += s.end - s.start;
}
+ return { clusters, totalSpeech };
+}
+
+function dropNegligibleClusters(segments) {
+ if (!segments?.length) return segments;
+ const { clusters, totalSpeech } = summarizeClusters(segments);
const keep = new Set(
- [...totals].filter(([, total]) => total >= minTotalSeconds).map(([speaker]) => speaker)
+ [...clusters]
+ .sort((a, b) => b[1].total - a[1].total)
+ .filter(([, { total, count }], rank) => {
+ if (total < MIN_CLUSTER_TOTAL_SECONDS) return false;
+ return (
+ rank < PROTECTED_CLUSTER_COUNT ||
+ total / totalSpeech >= PHANTOM_MAX_SHARE ||
+ total / count >= PHANTOM_MAX_MEAN_SEGMENT_SECONDS
+ );
+ })
+ .map(([speaker]) => speaker)
);
- if (keep.size === 0 || keep.size === totals.size) return segments;
+ if (keep.size === 0 || keep.size === clusters.size) return segments;
return segments.filter((s) => keep.has(s.speaker));
}
+function isCollapsedDiarization(segments) {
+ const { clusters } = summarizeClusters(segments);
+ const heard = [...clusters.values()].filter(({ total }) => total >= MIN_CLUSTER_TOTAL_SECONDS);
+ if (heard.length < 2) return false;
+ const [largest, ...rest] = heard.sort((a, b) => b.total - a.total);
+ const heardSpeech = heard.reduce((sum, { total }) => sum + total, 0);
+ return (
+ largest.total / heardSpeech > COLLAPSED_TOP_SHARE &&
+ rest.every(({ total, count }) => total / count < COLLAPSED_MAX_MEAN_SEGMENT_SECONDS)
+ );
+}
+
+// Keeps the `cap` largest clusters and relabels each extra one to the kept
+// speaker whose voice centroid it is most similar to. Folding extras into the
+// largest cluster (capSpeakerClusters) handed one person's turns to the other
+// whenever clustering split a real speaker. Clusters without a centroid still
+// fold into the largest.
+function capSpeakerClustersByVoice(segments, cap, centroids, similarity) {
+ const { clusters } = summarizeClusters(segments);
+ if (clusters.size <= cap) return segments;
+
+ const ranked = [...clusters].sort((a, b) => b[1].total - a[1].total).map(([speaker]) => speaker);
+ const kept = ranked.slice(0, cap);
+ const relabel = new Map();
+ for (const extra of ranked.slice(cap)) {
+ const voice = centroids.get(extra);
+ let target = kept[0];
+ let bestSimilarity = -Infinity;
+ if (voice) {
+ for (const speaker of kept) {
+ const keptVoice = centroids.get(speaker);
+ if (!keptVoice) continue;
+ const score = similarity(voice, keptVoice);
+ if (score > bestSimilarity) {
+ bestSimilarity = score;
+ target = speaker;
+ }
+ }
+ }
+ relabel.set(extra, target);
+ }
+ return segments.map((s) =>
+ relabel.has(s.speaker) ? { ...s, speaker: relabel.get(s.speaker) } : s
+ );
+}
+
module.exports = {
DEFAULT_CLUSTER_THRESHOLD,
LONG_AUDIO_CLUSTER_THRESHOLD,
@@ -60,4 +153,6 @@ module.exports = {
clusterThresholdForDuration,
resolveClusterThreshold,
dropNegligibleClusters,
+ isCollapsedDiarization,
+ capSpeakerClustersByVoice,
};
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index fad3de9c2..27ab76f0b 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -3777,14 +3777,16 @@ class IPCHandlers {
if (!realPath) return { success: false, error: "File path not allowed" };
filePath = realPath;
- const numSpeakers = Math.min(
- MAX_SPEAKER_COUNT,
- Math.max(-1, Math.round(Number(options.numSpeakers) || -1))
- );
+ const maxSpeakers = normalizeStoredSpeakerCount(options.numSpeakers) ?? MAX_SPEAKER_COUNT;
const { convertToWav } = require("./ffmpegUtils");
const { getSafeTempDir } = require("./safeTempDir");
- const { resolveClusterThreshold, dropNegligibleClusters } = require("./diarizationPolicy");
+ const {
+ resolveClusterThreshold,
+ dropNegligibleClusters,
+ isCollapsedDiarization,
+ capSpeakerClustersByVoice,
+ } = require("./diarizationPolicy");
const { PCM16_MONO_16K_BYTES_PER_SECOND } = require("./transcriptionTimeout");
const wavPath = path.join(getSafeTempDir(), `ow-diarize-${Date.now()}.wav`);
@@ -3798,22 +3800,48 @@ class IPCHandlers {
const durationSeconds = fs.statSync(wavPath).size / PCM16_MONO_16K_BYTES_PER_SECOND;
const threshold = resolveClusterThreshold(durationSeconds, options.threshold);
- let segments = await this.diarizationManager.diarize(wavPath, {
- numSpeakers,
- threshold,
- signal,
- });
+ // Always auto-cluster: forcing 2 clusters on #2021's two-person call
+ // split short utterances from long ones instead of one speaker from
+ // the other, merging both speakers. The requested count is applied
+ // as a cap after cleanup instead.
+ let segments = await this.diarizationManager.diarize(wavPath, { threshold, signal });
if (signal?.aborted) {
return { success: false, error: "Cancelled", code: "UPLOAD_CANCELLED" };
}
+ // A collapsed run's labels are wrong either way: two people under
+ // one label, or one person split off into a phantom. With no
+ // segments the renderer keeps the plain transcript and shows the
+ // diarization warning instead. Judged before phantoms are dropped;
+ // a request for one speaker is met by the cap below instead.
+ if (maxSpeakers > 1 && isCollapsedDiarization(segments)) {
+ debugLogger.warn("Discarding diarization: one cluster holds nearly all speech", {
+ durationSeconds,
+ });
+ segments = [];
+ }
// The meeting path caps clusters via its expectation resolver; this
// path fed raw sherpa output to the merge, which is how a 2-person
// voice memo surfaced 46 speakers.
segments = dropNegligibleClusters(segments);
- segments = this.diarizationManager.capSpeakerClusters(
- segments,
- numSpeakers > 0 ? numSpeakers : MAX_SPEAKER_COUNT
- );
+ if (new Set(segments.map((s) => s.speaker)).size > maxSpeakers) {
+ const speakerEmbeddings = require("./speakerEmbeddings");
+ let centroids = new Map();
+ try {
+ centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments, {
+ signal,
+ });
+ } catch (error) {
+ debugLogger.warn("Speaker voices unavailable; extra clusters fold into the largest", {
+ error: error.message,
+ });
+ }
+ if (signal?.aborted) {
+ return { success: false, error: "Cancelled", code: "UPLOAD_CANCELLED" };
+ }
+ segments = capSpeakerClustersByVoice(segments, maxSpeakers, centroids, (a, b) =>
+ speakerEmbeddings.cosineSimilarity(a, b)
+ );
+ }
// Callers persist this as audio_duration_seconds: for picked files
// the renderer has no other duration source.
return { success: true, segments, durationSeconds };
diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.js
index 736bed0dd..3103da62e 100644
--- a/src/helpers/speakerEmbeddings.js
+++ b/src/helpers/speakerEmbeddings.js
@@ -10,6 +10,11 @@ const MIN_SEGMENT_SECONDS = 1.5;
const MIN_SEGMENT_SAMPLES = SAMPLE_RATE * MIN_SEGMENT_SECONDS;
const MAX_EMBEDDING_SECONDS = 8;
const MAX_EMBEDDING_SAMPLES = SAMPLE_RATE * MAX_EMBEDDING_SECONDS;
+const BYTES_PER_SAMPLE = 2;
+// Same as the meeting path's per-speaker centroids.
+const CENTROID_SEGMENTS_PER_CLUSTER = 3;
+// Enough for the RIFF, fmt and metadata chunks ffmpeg writes before "data".
+const WAV_HEADER_READ_BYTES = 64 * 1024;
const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
// Live meetings extract during remote speech, so this normally fires once a meeting and its
// post-meeting diarization are done (a longer silence mid-meeting costs one reload). The
@@ -133,26 +138,85 @@ class SpeakerEmbeddings {
if (endSec - startSec < MIN_SEGMENT_SECONDS) return null;
const buf = fs.readFileSync(wavPath);
- const { sampleRate, dataOffset } = this._parseWavHeader(buf);
+ const { byteOffset, numSamples } = this._embeddingWindow(
+ this._parseWavHeader(buf),
+ startSec,
+ endSec
+ );
+ return this._extractEmbeddingFromSamples(this._decodePcm16(buf, byteOffset, numSamples));
+ }
+
+ // One voice per diarized cluster, from its longest segments. Reads only the
+ // audio each embedding uses: an upload's WAV runs about 115 MB an hour. A
+ // cluster whose extraction fails goes without a voice; the others keep theirs.
+ async extractClusterCentroids(wavPath, segments, { signal = null } = {}) {
+ const bySpeaker = new Map();
+ for (const segment of segments) {
+ if (segment.end - segment.start < MIN_SEGMENT_SECONDS) continue;
+ if (!bySpeaker.has(segment.speaker)) bySpeaker.set(segment.speaker, []);
+ bySpeaker.get(segment.speaker).push(segment);
+ }
+
+ const centroids = new Map();
+ const fd = fs.openSync(wavPath, "r");
+ try {
+ const header = this._parseWavHeader(this._readBytes(fd, 0, WAV_HEADER_READ_BYTES));
+ for (const [speaker, speakerSegments] of bySpeaker) {
+ if (signal?.aborted) break;
+ const longest = speakerSegments
+ .sort((a, b) => b.end - b.start - (a.end - a.start))
+ .slice(0, CENTROID_SEGMENTS_PER_CLUSTER);
+ try {
+ const embeddings = [];
+ for (const segment of longest) {
+ const { byteOffset, numSamples } = this._embeddingWindow(
+ header,
+ segment.start,
+ segment.end
+ );
+ const pcm = this._readBytes(fd, byteOffset, numSamples * BYTES_PER_SAMPLE);
+ const embedding = await this._extractEmbeddingFromSamples(
+ this._decodePcm16(pcm, 0, numSamples)
+ );
+ if (embedding) embeddings.push(embedding);
+ }
+ if (embeddings.length > 0) centroids.set(speaker, this.computeCentroid(embeddings));
+ } catch (error) {
+ debugLogger.warn("speaker-embeddings cluster voice unavailable", {
+ speaker,
+ error: error.message,
+ });
+ }
+ }
+ } finally {
+ fs.closeSync(fd);
+ }
+ return centroids;
+ }
+
+ _readBytes(fd, position, length) {
+ const buf = Buffer.alloc(length);
+ const bytesRead = fs.readSync(fd, buf, 0, length, position);
+ return buf.subarray(0, bytesRead);
+ }
+ // The last MAX_EMBEDDING_SECONDS of the segment, as a byte offset into the WAV.
+ _embeddingWindow({ sampleRate, dataOffset }, startSec, endSec) {
const cappedSeconds = Math.min(endSec - startSec, MAX_EMBEDDING_SECONDS);
- const cappedStartSec = endSec - cappedSeconds;
- const startSample = Math.floor(cappedStartSec * sampleRate);
- const endSample = Math.floor(endSec * sampleRate);
- const numSamples = endSample - startSample;
+ const startSample = Math.floor((endSec - cappedSeconds) * sampleRate);
+ const numSamples = Math.floor(endSec * sampleRate) - startSample;
+ return { byteOffset: dataOffset + startSample * BYTES_PER_SAMPLE, numSamples };
+ }
+ // Samples past the end of buf stay silent.
+ _decodePcm16(buf, offset, numSamples) {
const samples = new Float32Array(numSamples);
- const bytesPerSample = 2;
- const offset = dataOffset + startSample * bytesPerSample;
-
for (let i = 0; i < numSamples; i++) {
- const bytePos = offset + i * bytesPerSample;
+ const bytePos = offset + i * BYTES_PER_SAMPLE;
if (bytePos + 1 >= buf.length) break;
- const int16 = buf.readInt16LE(bytePos);
- samples[i] = int16 / 32768;
+ samples[i] = buf.readInt16LE(bytePos) / 32768;
}
-
- return this._extractEmbeddingFromSamples(samples);
+ return samples;
}
_parseWavHeader(buf) {
diff --git a/src/locales/ar/translation.json b/src/locales/ar/translation.json
index c174dcf03..82da25580 100644
--- a/src/locales/ar/translation.json
+++ b/src/locales/ar/translation.json
@@ -2231,7 +2231,7 @@
"speakerDetectionDescription": "تحديد المتحدثين المختلفين في الصوت",
"numSpeakersPlaceholder": "اكتشاف تلقائي",
"numSpeakersLabel": "عدد المتحدثين",
- "numSpeakersHint": "يؤدي تحديد العدد الدقيق إلى تحسين الدقة، خاصة في التسجيلات الطويلة.",
+ "numSpeakersHint": "يُكتشف المتحدثون تلقائيًا، بحد أقصى هذا العدد.",
"downloadingModels": "جاري تنزيل النماذج...",
"openaiDiarizeNote": "يستبدل نموذج التفريغ الذي اخترته بالنموذج gpt-4o-transcribe-diarize، وهو نموذج OpenAI الوحيد الذي يدعم اكتشاف المتحدثين. قد تختلف التكاليف عن نموذجك المحدد.",
"mistralDiarizeNote": "تم تضمين ميزة اكتشاف المتحدث بدون تكلفة إضافية مع Voxtral.",
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index c89939715..406a0a76f 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -3433,7 +3433,7 @@
"speakerDetectionDescription": "Verschiedene Sprecher im Audio identifizieren",
"numSpeakersPlaceholder": "Automatisch erkennen",
"numSpeakersLabel": "Anzahl der Sprecher",
- "numSpeakersHint": "Die genaue Anzahl verbessert die Genauigkeit, besonders bei langen Aufnahmen.",
+ "numSpeakersHint": "Sprecher werden automatisch erkannt, bis zu dieser Anzahl.",
"downloadingModels": "Modelle werden heruntergeladen...",
"openaiDiarizeNote": "Überschreibt Ihr Transkriptionsmodell mit gpt-4o-transcribe-diarize, dem einzigen OpenAI-Modell mit Sprechererkennung. Die Kosten können von Ihrem gewählten Modell abweichen.",
"mistralDiarizeNote": "Sprechererkennung ohne Zusatzkosten mit Voxtral enthalten.",
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index 1dcdb078e..7990787a0 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -3433,7 +3433,7 @@
"speakerDetectionDescription": "Identify different speakers in the audio",
"numSpeakersPlaceholder": "Auto-detect",
"numSpeakersLabel": "Number of speakers",
- "numSpeakersHint": "Setting the exact count improves accuracy, especially on long recordings.",
+ "numSpeakersHint": "Speakers are detected automatically, up to this number.",
"downloadingModels": "Downloading models...",
"openaiDiarizeNote": "Overrides your transcription model with gpt-4o-transcribe-diarize, the only OpenAI model with speaker detection. Costs may differ from your selected model.",
"mistralDiarizeNote": "Speaker detection included at no extra cost with Voxtral.",
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 4c7835468..90e24f75d 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -3134,7 +3134,7 @@
"speakerDetectionDescription": "Identificar diferentes hablantes en el audio",
"numSpeakersPlaceholder": "Detección automática",
"numSpeakersLabel": "Número de hablantes",
- "numSpeakersHint": "Indicar el número exacto mejora la precisión, sobre todo en grabaciones largas.",
+ "numSpeakersHint": "Los hablantes se detectan automáticamente, hasta este número.",
"downloadingModels": "Descargando modelos...",
"openaiDiarizeNote": "Reemplaza tu modelo de transcripción con gpt-4o-transcribe-diarize, el único modelo de OpenAI con detección de hablantes. Los costos pueden diferir de tu modelo seleccionado.",
"mistralDiarizeNote": "Detección de hablantes incluida sin costo adicional con Voxtral.",
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index 2bd29242f..8aaeaf708 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -3433,7 +3433,7 @@
"speakerDetectionDescription": "Identifier les différents intervenants dans l'audio",
"numSpeakersPlaceholder": "Détection automatique",
"numSpeakersLabel": "Nombre d'intervenants",
- "numSpeakersHint": "Indiquer le nombre exact améliore la précision, surtout pour les enregistrements longs.",
+ "numSpeakersHint": "Les intervenants sont détectés automatiquement, jusqu'à ce nombre.",
"downloadingModels": "Téléchargement des modèles...",
"openaiDiarizeNote": "Remplace votre modèle de transcription par gpt-4o-transcribe-diarize, le seul modèle OpenAI avec détection des locuteurs. Les coûts peuvent différer de votre modèle sélectionné.",
"mistralDiarizeNote": "Détection des locuteurs incluse sans frais supplémentaires avec Voxtral.",
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index 323dff24d..458c8245d 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -3081,7 +3081,7 @@
"speakerDetectionDescription": "Identifica i diversi parlanti nell'audio",
"numSpeakersPlaceholder": "Rilevamento automatico",
"numSpeakersLabel": "Numero di parlanti",
- "numSpeakersHint": "Indicare il numero esatto migliora la precisione, soprattutto nelle registrazioni lunghe.",
+ "numSpeakersHint": "I parlanti vengono rilevati automaticamente, fino a questo numero.",
"downloadingModels": "Download modelli in corso...",
"openaiDiarizeNote": "Sostituisce il tuo modello di trascrizione con gpt-4o-transcribe-diarize, l’unico modello OpenAI con rilevamento dei parlanti. I costi potrebbero differire dal modello selezionato.",
"mistralDiarizeNote": "Rilevamento dei parlanti incluso senza costi aggiuntivi con Voxtral.",
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index 2292c8c04..fe012dc53 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -3148,7 +3148,7 @@
"speakerDetectionDescription": "音声内の異なる話者を識別",
"numSpeakersPlaceholder": "自動検出",
"numSpeakersLabel": "話者の人数",
- "numSpeakersHint": "正確な人数を設定すると、特に長い録音で精度が向上します。",
+ "numSpeakersHint": "話者はこの人数を上限として自動で検出されます。",
"downloadingModels": "モデルをダウンロード中...",
"openaiDiarizeNote": "話者検出をサポートする唯一のOpenAIモデルであるgpt-4o-transcribe-diarizeに切り替わります。選択したモデルとコストが異なる場合があります。",
"mistralDiarizeNote": "Voxtralでの話者検出は追加料金なしで利用できます。",
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index 985e120a5..a67737809 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -3081,7 +3081,7 @@
"speakerDetectionDescription": "Identificar diferentes locutores no áudio",
"numSpeakersPlaceholder": "Detecção automática",
"numSpeakersLabel": "Número de falantes",
- "numSpeakersHint": "Definir o número exato melhora a precisão, principalmente em gravações longas.",
+ "numSpeakersHint": "Os falantes são detectados automaticamente, até este número.",
"downloadingModels": "Baixando modelos...",
"openaiDiarizeNote": "Substitui seu modelo de transcrição por gpt-4o-transcribe-diarize, o único modelo OpenAI com detecção de falantes. Os custos podem diferir do modelo selecionado.",
"mistralDiarizeNote": "Detecção de falantes incluída sem custo adicional com Voxtral.",
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index c2e198291..c7e06fb54 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -3101,7 +3101,7 @@
"speakerDetectionDescription": "Определить разных говорящих в аудио",
"numSpeakersPlaceholder": "Автоопределение",
"numSpeakersLabel": "Количество говорящих",
- "numSpeakersHint": "Точное число повышает точность, особенно для длинных записей.",
+ "numSpeakersHint": "Говорящие определяются автоматически, но не больше этого числа.",
"downloadingModels": "Загрузка моделей...",
"openaiDiarizeNote": "Заменяет вашу модель транскрипции на gpt-4o-transcribe-diarize — единственную модель OpenAI с определением говорящих. Стоимость может отличаться от выбранной модели.",
"mistralDiarizeNote": "Определение говорящих включено без дополнительной оплаты с Voxtral.",
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index fd4e8de1a..562eaa93d 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -3081,7 +3081,7 @@
"speakerDetectionDescription": "识别音频中的不同说话人",
"numSpeakersPlaceholder": "自动检测",
"numSpeakersLabel": "说话人数",
- "numSpeakersHint": "设置准确的人数可以提高准确性,尤其是长录音。",
+ "numSpeakersHint": "将自动检测说话人,不超过此人数。",
"downloadingModels": "正在下载模型...",
"openaiDiarizeNote": "将替换您的转录模型为 gpt-4o-transcribe-diarize,这是唯一支持说话人检测的 OpenAI 模型。费用可能与您选择的模型不同。",
"mistralDiarizeNote": "使用 Voxtral 时说话人检测无需额外费用。",
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index 5ec7d8394..f563d953f 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -3081,7 +3081,7 @@
"speakerDetectionDescription": "辨識音訊中的不同說話者",
"numSpeakersPlaceholder": "自動偵測",
"numSpeakersLabel": "說話人數",
- "numSpeakersHint": "設定準確的人數可以提高準確度,尤其是長錄音。",
+ "numSpeakersHint": "將自動偵測說話人,不超過此人數。",
"downloadingModels": "正在下載模型...",
"openaiDiarizeNote": "將替換您的轉錄模型為 gpt-4o-transcribe-diarize,這是唯一支援說話人偵測的 OpenAI 模型。費用可能與您選擇的模型不同。",
"mistralDiarizeNote": "使用 Voxtral 時說話人偵測無需額外費用。",
diff --git a/src/services/uploadNotes.ts b/src/services/uploadNotes.ts
index ac68bd4ff..d21225c55 100644
--- a/src/services/uploadNotes.ts
+++ b/src/services/uploadNotes.ts
@@ -1,16 +1,5 @@
-import { MAX_SPEAKER_COUNT } from "../constants/speakerDetection.json";
import type { DiarizationSettings } from "./fileTranscription";
-// Renderer twin of normalizeStoredSpeakerCount in ../helpers/speakerCount.js.
-// That file is CommonJS for the main process (database.js, ipcHandlers.js), and
-// renderer source cannot load CJS modules under Vite — test/services/
-// uploadNotes.test.js holds the two implementations to identical outputs.
-function normalizeSpeakerCount(value: number | null | undefined): number | null {
- const count = Number(value);
- if (!Number.isInteger(count) || count < 1) return null;
- return Math.min(count, MAX_SPEAKER_COUNT);
-}
-
export interface UploadSegment {
text: string;
start: number;
@@ -44,15 +33,14 @@ export function buildUploadTranscript(
return stored.length ? JSON.stringify(stored) : undefined;
}
-// What an upload persists onto the note row about the diarizer invocation,
+// What an upload persists onto the note row about the diarization run,
// matching the meeting path's write semantics: the columns are written only
// when diarization ran. A null diarization_enabled means "user never chose" and
// consumers fall back to the global speaker setting — writing 0 would force
-// diarization off when recording into the note later. The
-// expected_speaker_count is only ever a count the diarizer was actually invoked
-// with (numSpeakers lingers in localStorage while its input is hidden); auto
-// detection stays null so isExplicitSpeakerCount never mistakes it for an
-// explicit choice.
+// diarization off when recording into the note later. The speaker count the
+// user entered is never stored: the upload applied it as a cap on detected
+// speakers, but recording into the note forces expected_speaker_count as the
+// exact count, so a generous cap would split that meeting into phantoms.
export function buildUploadNoteMetadata(
diarization: DiarizationSettings,
durationSeconds?: number | null,
@@ -63,12 +51,7 @@ export function buildUploadNoteMetadata(
const noteUpdates: Record<string, unknown> | null =
diarization.enabled || transcript
? {
- ...(diarization.enabled
- ? {
- diarization_enabled: 1,
- expected_speaker_count: normalizeSpeakerCount(diarization.numSpeakers),
- }
- : {}),
+ ...(diarization.enabled ? { diarization_enabled: 1 } : {}),
...(transcript ? { transcript } : {}),
}
: null;
diff --git a/test/helpers/diarizationPolicy.test.js b/test/helpers/diarizationPolicy.test.js
index 0573566aa..6243ac614 100644
--- a/test/helpers/diarizationPolicy.test.js
+++ b/test/helpers/diarizationPolicy.test.js
@@ -10,6 +10,8 @@ const {
clusterThresholdForDuration,
resolveClusterThreshold,
dropNegligibleClusters,
+ isCollapsedDiarization,
+ capSpeakerClustersByVoice,
} = require("../../src/helpers/diarizationPolicy");
// 0.55 is tuned for short clean audio; on a 73-minute single-mic voice memo one
@@ -42,6 +44,12 @@ test("hour-plus audio is clamped to the long-audio threshold", () => {
assert.equal(clusterThresholdForDuration(3 * 3600), LONG_AUDIO_CLUSTER_THRESHOLD);
});
+// Too high a threshold is the opposite failure: at 0.72 and above, a
+// 20-minute two-person call (#2021) came back as one speaker plus phantoms.
+test("the long-audio threshold stays below where #2021's two speakers merged", () => {
+ assert.ok(clusterThresholdForDuration(3 * 3600) < 0.72);
+});
+
test("unknown duration falls back to the default threshold", () => {
assert.equal(clusterThresholdForDuration(NaN), DEFAULT_CLUSTER_THRESHOLD);
assert.equal(clusterThresholdForDuration(undefined), DEFAULT_CLUSTER_THRESHOLD);
@@ -73,7 +81,7 @@ test("clusters below the minimum total speaking time are dropped", () => {
);
});
-test("a cluster whose short segments add up past the minimum survives", () => {
+test("the second-largest cluster survives once its short segments pass the minimum", () => {
const segments = [
{ start: 0, end: 30, speaker: "speaker_0" },
{ start: 30, end: 30.6, speaker: "speaker_1" },
@@ -82,6 +90,95 @@ test("a cluster whose short segments add up past the minimum survives", () => {
assert.equal(dropNegligibleClusters(segments).length, 3);
});
+// `count` segments of `seconds` each, so a fixture can reproduce a cluster's
+// measured total speech and mean segment length.
+function cluster(speaker, count, seconds) {
+ return Array.from({ length: count }, (_, i) => ({
+ start: i * 20,
+ end: i * 20 + seconds,
+ speaker,
+ }));
+}
+
+const speakersOf = (segments) => [...new Set(segments.map((s) => s.speaker))].sort();
+
+// Measured on a 20-minute two-person Spanish call (#2021): four phantom
+// clusters of short backchannels and turn starts, each over the 1 s floor.
+test("short-utterance phantom clusters are dropped from a two-person call", () => {
+ const segments = [
+ ...cluster("speaker_06", 79, 6.46),
+ ...cluster("speaker_01", 85, 5.44),
+ ...cluster("speaker_00", 23, 1.72),
+ ...cluster("speaker_02", 20, 1.57),
+ ...cluster("speaker_04", 9, 1.49),
+ ...cluster("speaker_03", 2, 0.72),
+ ];
+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_01", "speaker_06"]);
+});
+
+// The same call clustered at 0.65: the larger phantom held 5.6 % of the speech
+// (59.6 s of 1058.4 s) in segments averaging 1.92 s.
+test("a short-segment phantom under 10% of the speech is dropped", () => {
+ const segments = [
+ ...cluster("speaker_0", 94, 5.77),
+ ...cluster("speaker_1", 83, 5.44),
+ ...cluster("speaker_2", 31, 1.92),
+ ...cluster("speaker_3", 3, 1.7),
+ ];
+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0", "speaker_1"]);
+});
+
+test("a minor third speaker with real turns survives", () => {
+ const segments = [
+ ...cluster("speaker_0", 80, 6),
+ ...cluster("speaker_1", 80, 6),
+ ...cluster("speaker_2", 7, 6),
+ ...cluster("speaker_3", 20, 1.5),
+ ];
+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), [
+ "speaker_0",
+ "speaker_1",
+ "speaker_2",
+ ]);
+});
+
+test("a short-turn speaker with a real share of the speech survives", () => {
+ const segments = [
+ ...cluster("speaker_0", 100, 2),
+ ...cluster("speaker_1", 100, 2),
+ ...cluster("speaker_2", 40, 2),
+ ];
+ assert.equal(speakersOf(dropNegligibleClusters(segments)).length, 3);
+});
+
+test("the second-largest cluster survives even when it looks like a phantom", () => {
+ const segments = [
+ ...cluster("speaker_0", 200, 6),
+ ...cluster("speaker_1", 20, 1.5),
+ ...cluster("speaker_2", 10, 1.5),
+ ];
+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0", "speaker_1"]);
+});
+
+// Forcing two clusters on #2021's call put both real speakers in one cluster
+// and kept a phantom for the rest.
+test("one cluster holding nearly all the speech is a collapsed run", () => {
+ const segments = [...cluster("speaker_01", 165, 6.06), ...cluster("speaker_00", 35, 1.72)];
+ assert.equal(isCollapsedDiarization(segments), true);
+});
+
+test("a balanced run or a single cluster is not a collapsed run", () => {
+ const balanced = [...cluster("speaker_06", 79, 6.46), ...cluster("speaker_01", 85, 5.44)];
+ assert.equal(isCollapsedDiarization(balanced), false);
+ assert.equal(isCollapsedDiarization(cluster("speaker_0", 50, 6)), false);
+ assert.equal(isCollapsedDiarization([]), false);
+});
+
+test("a dominant speaker beside someone with real turns is not a collapsed run", () => {
+ const lecture = [...cluster("speaker_0", 550, 6), ...cluster("speaker_1", 5, 6)];
+ assert.equal(isCollapsedDiarization(lecture), false);
+});
+
test("dropping never empties the result or touches a clean input", () => {
const allTiny = [
{ start: 0, end: 0.3, speaker: "speaker_0" },
@@ -98,3 +195,169 @@ test("dropping never empties the result or touches a clean input", () => {
test("minimum speaking time constant is sane", () => {
assert.ok(MIN_CLUSTER_TOTAL_SECONDS >= 0.5 && MIN_CLUSTER_TOTAL_SECONDS <= 5);
});
+
+// sherpa numbers clusters by first appearance, and on #2021's call the first
+// cluster (speaker_00) was a phantom. Protection must follow size, not order.
+test("the protected clusters are the largest, whatever order sherpa lists them in", () => {
+ const segments = [
+ ...cluster("speaker_00", 23, 1.72),
+ ...cluster("speaker_01", 85, 5.44),
+ ...cluster("speaker_02", 20, 1.57),
+ ...cluster("speaker_06", 79, 6.46),
+ ];
+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_01", "speaker_06"]);
+});
+
+test("the 1 s floor applies to the protected second-largest cluster", () => {
+ const segments = [...cluster("speaker_0", 5, 6), ...cluster("speaker_1", 2, 0.4)];
+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0"]);
+});
+
+test("a short-segment cluster at exactly 10% of the speech is kept", () => {
+ const segments = [
+ ...cluster("speaker_0", 45, 10),
+ ...cluster("speaker_1", 45, 10),
+ ...cluster("speaker_2", 50, 2),
+ ];
+ assert.equal(speakersOf(dropNegligibleClusters(segments)).length, 3);
+});
+
+test("a small cluster is kept from a 3 s mean segment up and dropped below it", () => {
+ const withMean = (seconds, count) => [
+ ...cluster("speaker_0", 47, 10),
+ ...cluster("speaker_1", 47, 10),
+ ...cluster("speaker_2", count, seconds),
+ ];
+ assert.equal(speakersOf(dropNegligibleClusters(withMean(3, 20))).length, 3);
+ assert.equal(speakersOf(dropNegligibleClusters(withMean(4, 15))).length, 3);
+ assert.equal(speakersOf(dropNegligibleClusters(withMean(2.5, 20))).length, 2);
+});
+
+// Documented cost of the phantom rule: by duration alone, a real third
+// participant with under 10 % of the speech in short turns looks like a
+// phantom. Flip this deliberately if the rule learns to use voices.
+test("characterization: a quiet third participant in short turns is dropped", () => {
+ const segments = [
+ ...cluster("speaker_0", 60, 7.5),
+ ...cluster("speaker_1", 60, 7.5),
+ ...cluster("speaker_2", 34, 2.8),
+ ];
+ assert.deepEqual(speakersOf(dropNegligibleClusters(segments)), ["speaker_0", "speaker_1"]);
+});
+
+test("a collapsed run is found whatever order sherpa lists its clusters in", () => {
+ const segments = [...cluster("speaker_00", 35, 1.72), ...cluster("speaker_01", 165, 6.06)];
+ assert.equal(isCollapsedDiarization(segments), true);
+});
+
+test("a run is collapsed only above 93% for the largest cluster", () => {
+ const withTop = (seconds) => [
+ ...cluster("speaker_0", 1, seconds),
+ ...cluster("speaker_1", 35, 2),
+ ];
+ assert.equal(isCollapsedDiarization(withTop(930)), false);
+ assert.equal(isCollapsedDiarization(withTop(931)), true);
+});
+
+// Short turns alone don't make a collapse: someone holding 15 % of the speech
+// in quick replies is a real second speaker.
+test("a lopsided conversation in short turns on both sides is not collapsed", () => {
+ const segments = [...cluster("speaker_0", 425, 2), ...cluster("speaker_1", 75, 2)];
+ assert.equal(isCollapsedDiarization(segments), false);
+});
+
+test("a run is not collapsed while any other cluster averages 2.2 s segments or more", () => {
+ const beside = (seconds) => [
+ ...cluster("speaker_0", 100, 10),
+ ...cluster("speaker_1", 1, seconds),
+ ];
+ assert.equal(isCollapsedDiarization(beside(2.2)), false);
+ assert.equal(isCollapsedDiarization(beside(2.19)), true);
+
+ const oneRealBesidePhantom = [
+ ...cluster("speaker_0", 91, 10),
+ ...cluster("speaker_1", 10, 5),
+ ...cluster("speaker_2", 20, 2),
+ ];
+ assert.equal(isCollapsedDiarization(oneRealBesidePhantom), false);
+});
+
+// The 1 s floor exists for these blips; a monologue must keep its label.
+test("a single speaker beside a sub-second blip is not collapsed", () => {
+ const monologue = [...cluster("speaker_0", 100, 6), ...cluster("speaker_1", 1, 0.45)];
+ assert.equal(isCollapsedDiarization(monologue), false);
+
+ const blips = [
+ ...cluster("speaker_0", 100, 6),
+ ...cluster("speaker_1", 1, 0.5),
+ ...cluster("speaker_2", 2, 0.4),
+ ];
+ assert.equal(isCollapsedDiarization(blips), false);
+});
+
+test("interviews, podcasts and lectures dominated by one voice keep their labels", () => {
+ const oralHistory = [...cluster("speaker_0", 40, 60), ...cluster("speaker_1", 40, 2.5)];
+ const podcast = [...cluster("speaker_0", 60, 20), ...cluster("speaker_1", 40, 2.9)];
+ const lecture = [...cluster("speaker_0", 500, 6), ...cluster("speaker_1", 6, 2.8)];
+ for (const run of [oralHistory, podcast, lecture]) {
+ assert.equal(isCollapsedDiarization(run), false);
+ }
+});
+
+test("characterization: a listener who only backchannels beside a dominant speaker reads as collapsed", () => {
+ const segments = [...cluster("speaker_0", 100, 10), ...cluster("speaker_1", 30, 1.8)];
+ assert.equal(isCollapsedDiarization(segments), true);
+});
+
+const dot = (a, b) => a.reduce((sum, value, i) => sum + value * b[i], 0);
+
+test("an extra cluster is relabelled to the kept speaker it sounds like", () => {
+ const segments = [
+ ...cluster("speaker_0", 10, 10),
+ ...cluster("speaker_1", 8, 10),
+ ...cluster("speaker_2", 6, 10),
+ ];
+ const centroids = new Map([
+ ["speaker_0", [1, 0]],
+ ["speaker_1", [0, 1]],
+ ["speaker_2", [0.1, 0.9]],
+ ]);
+ const capped = capSpeakerClustersByVoice(segments, 2, centroids, dot);
+ assert.deepEqual(speakersOf(capped), ["speaker_0", "speaker_1"]);
+ assert.equal(capped.filter((s) => s.speaker === "speaker_1").length, 14);
+ assert.deepEqual(
+ capped.slice(0, 18).map((s) => s.speaker),
+ segments.slice(0, 18).map((s) => s.speaker)
+ );
+});
+
+test("an extra cluster without a usable voice folds into the largest", () => {
+ const segments = [
+ ...cluster("speaker_0", 10, 10),
+ ...cluster("speaker_1", 8, 10),
+ ...cluster("speaker_2", 6, 10),
+ ];
+ const noExtraVoice = new Map([
+ ["speaker_0", [1, 0]],
+ ["speaker_1", [0, 1]],
+ ]);
+ const capped = capSpeakerClustersByVoice(segments, 2, noExtraVoice, dot);
+ assert.equal(capped.filter((s) => s.speaker === "speaker_0").length, 16);
+
+ // A kept speaker without a voice can't be matched, so the extra goes to the
+ // closest kept speaker that has one.
+ const onlySecondVoice = new Map([
+ ["speaker_1", [0, 1]],
+ ["speaker_2", [1, 0]],
+ ]);
+ const matched = capSpeakerClustersByVoice(segments, 2, onlySecondVoice, dot);
+ assert.equal(matched.filter((s) => s.speaker === "speaker_1").length, 14);
+});
+
+test("the voice cap leaves a run within the cap untouched and can fold everything into one", () => {
+ const segments = [...cluster("speaker_0", 10, 10), ...cluster("speaker_1", 8, 10)];
+ assert.equal(capSpeakerClustersByVoice(segments, 2, new Map(), dot), segments);
+ assert.deepEqual(speakersOf(capSpeakerClustersByVoice(segments, 1, new Map(), dot)), [
+ "speaker_0",
+ ]);
+});
diff --git a/test/helpers/diarizeAudioFileHandler.test.js b/test/helpers/diarizeAudioFileHandler.test.js
new file mode 100644
index 000000000..3959015d2
--- /dev/null
+++ b/test/helpers/diarizeAudioFileHandler.test.js
@@ -0,0 +1,244 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const Module = require("node:module");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const handlersModulePath = require.resolve("../../src/helpers/ipcHandlers");
+const originalLoad = Module._load;
+const handlers = new Map();
+
+const electronStub = {
+ app: {
+ getPath: () => os.tmpdir(),
+ getName: () => "test",
+ getVersion: () => "0.0.0",
+ isPackaged: false,
+ on: () => {},
+ requestSingleInstanceLock: () => true,
+ },
+ ipcMain: {
+ handle: (channel, fn) => handlers.set(channel, fn),
+ on: () => {},
+ removeHandler: () => {},
+ },
+ net: {},
+ BrowserWindow: class BrowserWindow {
+ static getAllWindows() {
+ return [];
+ }
+ static fromWebContents() {
+ return null;
+ }
+ },
+ shell: {},
+ dialog: {},
+ screen: { getPrimaryDisplay: () => ({ workAreaSize: { width: 0, height: 0 } }) },
+ systemPreferences: { getMediaAccessStatus: () => "granted" },
+ session: { fromPartition: () => ({}) },
+ clipboard: {},
+ nativeImage: {},
+ globalShortcut: {},
+ utilityProcess: {},
+ MessageChannelMain: class {},
+};
+
+// Voices are looked up by cluster id; tests set the centroids (or make the
+// lookup fail) per case.
+let centroidCalls = [];
+let clusterCentroids = async () => new Map();
+const speakerEmbeddingsStub = {
+ extractClusterCentroids: (...args) => {
+ centroidCalls.push(args);
+ return clusterCentroids(...args);
+ },
+ cosineSimilarity: (a, b) => a.reduce((sum, value, i) => sum + value * b[i], 0),
+};
+
+// ffmpeg is not run: the handler only needs a WAV whose size gives a duration.
+Module._load = function loadWithMocks(request, parent, isMain) {
+ if (request === "electron") return electronStub;
+ if (parent?.filename === handlersModulePath && request === "./ffmpegUtils") {
+ return {
+ ...originalLoad.call(this, request, parent, isMain),
+ convertToWav: async (_input, output) => fs.writeFileSync(output, Buffer.alloc(32000)),
+ };
+ }
+ if (parent?.filename === handlersModulePath && request === "./speakerEmbeddings") {
+ return speakerEmbeddingsStub;
+ }
+ return originalLoad.call(this, request, parent, isMain);
+};
+
+function anything() {
+ return new Proxy(function () {}, {
+ get: (t, prop) => {
+ if (prop === Symbol.toPrimitive || prop === "toString") return () => "";
+ if (prop === "then") return undefined;
+ return anything();
+ },
+ apply: () => anything(),
+ });
+}
+
+let diarizeCalls = [];
+let diarizerOutput = [];
+let audioPath;
+let handler;
+
+test.before(() => {
+ delete require.cache[handlersModulePath];
+ const IPCHandlers = require(handlersModulePath);
+ const Ctor = IPCHandlers.default || IPCHandlers;
+ const target = {
+ _uploadCancelRegistry: { register: () => ({ signal: undefined, release: () => {} }) },
+ diarizationManager: {
+ isModelDownloaded: () => true,
+ diarize: async (_wavPath, options) => {
+ diarizeCalls.push(options);
+ return diarizerOutput;
+ },
+ },
+ };
+ Ctor.prototype.setupHandlers.call(
+ new Proxy(target, { get: (t, prop) => (prop in t ? t[prop] : anything()) })
+ );
+ handler = handlers.get("diarize-audio-file");
+ assert.ok(handler, "diarize-audio-file must be registered");
+
+ audioPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "ow-diarize-test-")), "call.mp3");
+ fs.writeFileSync(audioPath, "");
+ Ctor.prototype.approveAudioPath(audioPath);
+});
+
+test.beforeEach(() => {
+ diarizeCalls = [];
+ centroidCalls = [];
+ clusterCentroids = async () => new Map();
+});
+
+test.after(() => {
+ Module._load = originalLoad;
+ fs.rmSync(path.dirname(audioPath), { recursive: true, force: true });
+});
+
+function cluster(speaker, count, seconds) {
+ return Array.from({ length: count }, (_, i) => ({
+ start: i * 20,
+ end: i * 20 + seconds,
+ speaker,
+ }));
+}
+
+const speakersOf = (segments) => [...new Set(segments.map((s) => s.speaker))].sort();
+
+// Forcing sherpa to 2 clusters on #2021's two-person call split short
+// utterances from long ones, merging both real speakers and keeping a phantom.
+// The requested count must never reach sherpa.
+test("a requested speaker count still clusters automatically", async () => {
+ diarizerOutput = [
+ ...cluster("speaker_06", 79, 6.46),
+ ...cluster("speaker_01", 85, 5.44),
+ ...cluster("speaker_00", 23, 1.72),
+ ];
+
+ const result = await handler({}, audioPath, { numSpeakers: 2 });
+
+ assert.equal(result.success, true);
+ assert.equal(diarizeCalls.length, 1);
+ // diarize() maps a missing count to --clustering.num-clusters=-1 (auto); its
+ // default doesn't replace null, which would reach sherpa as "null".
+ assert.equal(Object.hasOwn(diarizeCalls[0], "numSpeakers"), false);
+ assert.deepEqual(speakersOf(result.segments), ["speaker_01", "speaker_06"]);
+ assert.equal(centroidCalls.length, 0, "voices are only read when clusters exceed the cap");
+});
+
+test("a collapsed run returns no segments, so the plain transcript is kept", async () => {
+ diarizerOutput = [...cluster("speaker_01", 165, 6.06), ...cluster("speaker_00", 35, 1.72)];
+
+ const result = await handler({}, audioPath, {});
+
+ assert.equal(result.success, true);
+ assert.deepEqual(result.segments, []);
+ assert.equal(result.durationSeconds, 1);
+});
+
+test("a requested speaker count caps the clusters automatic clustering finds", async () => {
+ diarizerOutput = [
+ ...cluster("speaker_0", 80, 6),
+ ...cluster("speaker_1", 60, 6),
+ ...cluster("speaker_2", 40, 6),
+ ];
+
+ const result = await handler({}, audioPath, { numSpeakers: 2 });
+
+ assert.deepEqual(speakersOf(result.segments), ["speaker_0", "speaker_1"]);
+});
+
+test("an extra cluster goes to the requested speaker whose voice it matches", async () => {
+ diarizerOutput = [
+ ...cluster("speaker_0", 80, 6),
+ ...cluster("speaker_1", 60, 6),
+ ...cluster("speaker_2", 40, 6),
+ ];
+ clusterCentroids = async () =>
+ new Map([
+ ["speaker_0", [1, 0]],
+ ["speaker_1", [0, 1]],
+ ["speaker_2", [0.2, 0.8]],
+ ]);
+
+ const result = await handler({}, audioPath, { numSpeakers: 2 });
+
+ assert.equal(centroidCalls.length, 1);
+ assert.equal(result.segments.filter((s) => s.speaker === "speaker_1").length, 100);
+ assert.equal(result.segments.filter((s) => s.speaker === "speaker_0").length, 80);
+});
+
+test("extra clusters fold into the largest when voices can't be read", async () => {
+ diarizerOutput = [
+ ...cluster("speaker_0", 80, 6),
+ ...cluster("speaker_1", 60, 6),
+ ...cluster("speaker_2", 40, 6),
+ ];
+ clusterCentroids = async () => {
+ throw new Error("Speaker embedding model not found");
+ };
+
+ const result = await handler({}, audioPath, { numSpeakers: 2 });
+
+ assert.equal(result.success, true);
+ assert.equal(result.segments.filter((s) => s.speaker === "speaker_0").length, 120);
+});
+
+// Dropping the 3 % phantom first would lift the interviewee from 88 % to
+// 90.7 % of the speech and discard a correctly labelled interview.
+test("a solo recording with a stray blip keeps its one speaker", async () => {
+ diarizerOutput = [...cluster("speaker_0", 100, 6), ...cluster("speaker_1", 1, 0.45)];
+
+ const result = await handler({}, audioPath, {});
+
+ assert.deepEqual(speakersOf(result.segments), ["speaker_0"]);
+});
+
+test("an interview with short questions keeps its labels", async () => {
+ diarizerOutput = [
+ ...cluster("speaker_0", 88, 10),
+ ...cluster("speaker_1", 36, 2.5),
+ ...cluster("speaker_2", 20, 1.5),
+ ];
+
+ const result = await handler({}, audioPath, {});
+
+ assert.deepEqual(speakersOf(result.segments), ["speaker_0", "speaker_1"]);
+});
+
+test("a request for one speaker labels one speaker instead of discarding the run", async () => {
+ diarizerOutput = [...cluster("speaker_01", 165, 6.06), ...cluster("speaker_00", 35, 1.72)];
+
+ const result = await handler({}, audioPath, { numSpeakers: 1 });
+
+ assert.equal(result.segments.length, 200);
+ assert.deepEqual(speakersOf(result.segments), ["speaker_01"]);
+});
diff --git a/test/helpers/speakerEmbeddingsClusterCentroids.test.js b/test/helpers/speakerEmbeddingsClusterCentroids.test.js
new file mode 100644
index 000000000..72f16dfb7
--- /dev/null
+++ b/test/helpers/speakerEmbeddingsClusterCentroids.test.js
@@ -0,0 +1,183 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const Module = require("node:module");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const speakerModulePath = require.resolve("../../src/helpers/speakerEmbeddings");
+const originalLoad = Module._load;
+Module._load = function loadWithMocks(request, parent, isMain) {
+ if (request === "electron") return { app: { getPath: () => os.tmpdir(), isPackaged: false } };
+ if (parent?.filename === speakerModulePath && request === "./onnxWorkerClient") return {};
+ return originalLoad.call(this, request, parent, isMain);
+};
+const speakerEmbeddings = require(speakerModulePath);
+Module._load = originalLoad;
+
+const SAMPLE_RATE = 16000;
+
+// A 16 kHz mono PCM16 WAV whose level changes every 10 s, so an embedding
+// stub can tell which region of the file it was handed.
+function writeWav(levels) {
+ const samplesPerRegion = SAMPLE_RATE * 10;
+ const data = Buffer.alloc(levels.length * samplesPerRegion * 2);
+ levels.forEach((level, region) => {
+ for (let i = 0; i < samplesPerRegion; i++) {
+ data.writeInt16LE(Math.round(level * 32768), (region * samplesPerRegion + i) * 2);
+ }
+ });
+ const header = Buffer.alloc(44);
+ header.write("RIFF", 0);
+ header.writeUInt32LE(36 + data.length, 4);
+ header.write("WAVE", 8);
+ header.write("fmt ", 12);
+ header.writeUInt32LE(16, 16);
+ header.writeUInt16LE(1, 20);
+ header.writeUInt16LE(1, 22);
+ header.writeUInt32LE(SAMPLE_RATE, 24);
+ header.writeUInt32LE(SAMPLE_RATE * 2, 28);
+ header.writeUInt16LE(2, 32);
+ header.writeUInt16LE(16, 34);
+ header.write("data", 36);
+ header.writeUInt32LE(data.length, 40);
+
+ const wavPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "ow-centroids-")), "a.wav");
+ fs.writeFileSync(wavPath, Buffer.concat([header, data]));
+ return wavPath;
+}
+
+const segments = [
+ { start: 0, end: 4, speaker: "speaker_a" },
+ { start: 4, end: 6.5, speaker: "speaker_a" },
+ { start: 6.5, end: 8.5, speaker: "speaker_a" },
+ { start: 8.4, end: 10, speaker: "speaker_a" },
+ { start: 10, end: 12, speaker: "speaker_b" },
+ { start: 12, end: 13, speaker: "speaker_c" },
+];
+
+let extracted;
+let wavPath;
+
+test.before(() => {
+ wavPath = writeWav([0.25, 0.5]);
+ speakerEmbeddings._extractEmbeddingFromSamples = async (samples) => {
+ extracted.push(samples);
+ return new Float32Array(512).fill(samples[0]);
+ };
+});
+
+test.beforeEach(() => {
+ extracted = [];
+});
+
+test.after(() => {
+ fs.rmSync(path.dirname(wavPath), { recursive: true, force: true });
+});
+
+test("each cluster's voice comes from its three longest segments of 1.5 s or more", async () => {
+ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments);
+
+ assert.deepEqual([...centroids.keys()], ["speaker_a", "speaker_b"]);
+ assert.deepEqual(
+ extracted.map((samples) => samples.length / SAMPLE_RATE),
+ [4, 2.5, 2, 2]
+ );
+ assert.ok(Math.abs(centroids.get("speaker_a")[0] - 0.25) < 1e-4);
+ assert.ok(Math.abs(centroids.get("speaker_b")[0] - 0.5) < 1e-4);
+});
+
+test("only the audio each voice uses is read, never the whole WAV", async () => {
+ const originalReadFile = fs.readFileSync;
+ const originalRead = fs.readSync;
+ let wholeFileReads = 0;
+ const reads = [];
+ fs.readFileSync = (file, ...rest) => {
+ if (file === wavPath) wholeFileReads += 1;
+ return originalReadFile(file, ...rest);
+ };
+ fs.readSync = (fd, buffer, offset, length, position) => {
+ reads.push(length);
+ return originalRead(fd, buffer, offset, length, position);
+ };
+ const originalOpen = fs.openSync;
+ const originalClose = fs.closeSync;
+ let openFiles = 0;
+ fs.openSync = (...args) => {
+ openFiles += 1;
+ return originalOpen(...args);
+ };
+ fs.closeSync = (fd) => {
+ openFiles -= 1;
+ return originalClose(fd);
+ };
+ try {
+ await speakerEmbeddings.extractClusterCentroids(wavPath, segments);
+ } finally {
+ fs.readFileSync = originalReadFile;
+ fs.readSync = originalRead;
+ fs.openSync = originalOpen;
+ fs.closeSync = originalClose;
+ }
+ assert.equal(wholeFileReads, 0);
+ assert.equal(openFiles, 0);
+ // The header, then one window per embedded segment.
+ assert.deepEqual(
+ reads.slice(1).map((bytes) => bytes / (SAMPLE_RATE * 2)),
+ [4, 2.5, 2, 2]
+ );
+});
+
+test("a segment longer than 8 s is embedded from its last 8 s", async () => {
+ await speakerEmbeddings.extractClusterCentroids(wavPath, [
+ { start: 0, end: 12, speaker: "speaker_a" },
+ ]);
+
+ const [samples] = extracted;
+ assert.equal(samples.length / SAMPLE_RATE, 8);
+ assert.ok(Math.abs(samples[0] - 0.25) < 1e-4);
+ assert.ok(Math.abs(samples[samples.length - 1] - 0.5) < 1e-4);
+});
+
+test("a cluster whose voice fails leaves the other clusters' voices intact", async () => {
+ const workingExtract = speakerEmbeddings._extractEmbeddingFromSamples;
+ speakerEmbeddings._extractEmbeddingFromSamples = async (samples) => {
+ if (Math.abs(samples[0] - 0.25) < 1e-4) throw new Error("onnx worker request timeout");
+ return workingExtract(samples);
+ };
+ try {
+ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments);
+ assert.deepEqual([...centroids.keys()], ["speaker_b"]);
+ } finally {
+ speakerEmbeddings._extractEmbeddingFromSamples = workingExtract;
+ }
+});
+
+test("a cancel between clusters stops before the next cluster", async () => {
+ const controller = new AbortController();
+ const workingExtract = speakerEmbeddings._extractEmbeddingFromSamples;
+ speakerEmbeddings._extractEmbeddingFromSamples = async (samples) => {
+ controller.abort();
+ return workingExtract(samples);
+ };
+ try {
+ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments, {
+ signal: controller.signal,
+ });
+ assert.deepEqual([...centroids.keys()], ["speaker_a"]);
+ } finally {
+ speakerEmbeddings._extractEmbeddingFromSamples = workingExtract;
+ }
+});
+
+test("a cancelled upload stops before extracting voices", async () => {
+ const controller = new AbortController();
+ controller.abort();
+
+ const centroids = await speakerEmbeddings.extractClusterCentroids(wavPath, segments, {
+ signal: controller.signal,
+ });
+
+ assert.equal(centroids.size, 0);
+ assert.equal(extracted.length, 0);
+});
diff --git a/test/services/fileTranscriptionDiarizationWarning.test.js b/test/services/fileTranscriptionDiarizationWarning.test.js
index ff6dd3aa2..348b32432 100644
--- a/test/services/fileTranscriptionDiarizationWarning.test.js
+++ b/test/services/fileTranscriptionDiarizationWarning.test.js
@@ -75,6 +75,32 @@ test("keeps the transcript and warns when speaker labels cannot be merged", asyn
assert.equal(result.diarizationWarning, true);
});
+// The diarize handler returns no segments when it discards a collapsed run.
+test("warns and keeps the measured duration when diarization returns no segments", async (t) => {
+ const { window, transcribeFileWithSpeakers } = await loadFileTranscription(t);
+ let mergeCalls = 0;
+ window.electronAPI.diarizeAudioFile = async () => ({
+ success: true,
+ segments: [],
+ durationSeconds: 1188,
+ });
+ window.electronAPI.mergeSpeakerText = async () => {
+ mergeCalls += 1;
+ return { success: true, text: "Speaker 1: Plain transcript" };
+ };
+
+ const result = await transcribeFileWithSpeakers("/tmp/audio.wav", localConfig(), {
+ enabled: true,
+ localModelsReady: true,
+ numSpeakers: null,
+ });
+
+ assert.equal(mergeCalls, 0);
+ assert.equal(result.text, "Plain transcript");
+ assert.equal(result.diarizationWarning, true);
+ assert.equal(result.durationSeconds, 1188);
+});
+
test("does not warn when speaker labels are merged", async (t) => {
const { window, transcribeFileWithSpeakers } = await loadFileTranscription(t);
window.electronAPI.diarizeAudioFile = async () => ({
diff --git a/test/services/uploadDiarizationPersistence.test.js b/test/services/uploadDiarizationPersistence.test.js
index 213c36451..21dbccb8e 100644
--- a/test/services/uploadDiarizationPersistence.test.js
+++ b/test/services/uploadDiarizationPersistence.test.js
@@ -3,10 +3,9 @@ const assert = require("node:assert/strict");
const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");
// Regression for the upload/URL-ingest path dropping diarization metadata: the
-// batch queue ran the diarizer but persisted none of diarization_enabled,
-// expected_speaker_count, or audio_duration_seconds, so an uploaded note that
-// WAS diarized presented to the app as though it was not (the meeting path
-// persists all of them).
+// batch queue ran the diarizer but persisted neither diarization_enabled nor
+// audio_duration_seconds, so an uploaded note that WAS diarized presented to
+// the app as though it was not (the meeting path persists both).
const transcription = {
useLocalWhisper: true,
@@ -105,7 +104,7 @@ test("a diarized file upload persists the diarization metadata", async (t) => {
// The diarizer runs under the same cancellable requestId as the
// transcription, so one cancel-upload-transcription aborts both.
assert.equal(typeof calls.diarize[0].requestId, "string");
- assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1, expected_speaker_count: 2 }]]);
+ assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1 }]]);
});
test("a diarized URL ingest persists the diarization metadata", async (t) => {
@@ -141,7 +140,7 @@ test("a diarized URL ingest persists the diarization metadata", async (t) => {
assert.equal(sourceFile, "Board Meeting Recording");
assert.equal(audioDuration, 4359.87);
- assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1, expected_speaker_count: 2 }]]);
+ assert.deepEqual(calls.updateNote, [[7, { diarization_enabled: 1 }]]);
assert.deepEqual(calls.deletedTempFiles, ["/tmp/ow-url-download.m4a"]);
});
diff --git a/test/services/uploadNotes.test.js b/test/services/uploadNotes.test.js
index 39f1faeef..11279df1a 100644
--- a/test/services/uploadNotes.test.js
+++ b/test/services/uploadNotes.test.js
@@ -4,7 +4,7 @@ const assert = require("node:assert/strict");
const load = () => import("../../src/services/uploadNotes.ts");
const { MAX_SPEAKER_COUNT } = require("../../src/constants/speakerDetection.json");
-test("maps the diarizer invocation onto the note columns", async () => {
+test("maps the diarization settings onto the note columns", async () => {
const { buildUploadNoteMetadata } = await load();
const { audioDurationSeconds, noteUpdates } = buildUploadNoteMetadata(
@@ -13,30 +13,29 @@ test("maps the diarizer invocation onto the note columns", async () => {
);
assert.equal(audioDurationSeconds, 4359.87);
- assert.deepEqual(noteUpdates, { diarization_enabled: 1, expected_speaker_count: 2 });
+ assert.deepEqual(noteUpdates, { diarization_enabled: 1 });
});
-test("auto speaker detection stores no explicit count", async () => {
+// The upload caps detected speakers at the entered count, but recording into
+// the note later forces expected_speaker_count as the exact count.
+test("the speaker count entered for an upload is never stored on the note", async () => {
const { buildUploadNoteMetadata } = await load();
- const { noteUpdates } = buildUploadNoteMetadata(
- { enabled: true, localModelsReady: true, numSpeakers: null },
- 60
- );
-
- // isExplicitSpeakerCount treats any positive integer as the user's own
- // choice, so auto detection must land as null, never a default.
- assert.equal(noteUpdates.expected_speaker_count, null);
- assert.equal(noteUpdates.diarization_enabled, 1);
+ for (const numSpeakers of [null, 2, MAX_SPEAKER_COUNT]) {
+ const { noteUpdates } = buildUploadNoteMetadata(
+ { enabled: true, localModelsReady: true, numSpeakers },
+ 60
+ );
+ assert.equal(Object.hasOwn(noteUpdates, "expected_speaker_count"), false);
+ assert.equal(noteUpdates.diarization_enabled, 1);
+ }
});
test("disabled diarization writes no columns at all", async () => {
const { buildUploadNoteMetadata } = await load();
// A null diarization_enabled defers to the global speaker setting when the
- // user records into the note later (a 0 would force it off), and writing no
- // count keeps the numSpeakers value lingering in localStorage while its
- // input is hidden from being stamped onto the note as an explicit choice.
+ // user records into the note later (a 0 would force it off).
const { audioDurationSeconds, noteUpdates } = buildUploadNoteMetadata(
{ enabled: false, localModelsReady: false, numSpeakers: 3 },
120
@@ -46,22 +45,6 @@ test("disabled diarization writes no columns at all", async () => {
assert.equal(audioDurationSeconds, 120);
});
-test("speaker counts reuse the stored-count clamp", async () => {
- const { buildUploadNoteMetadata } = await load();
-
- const above = buildUploadNoteMetadata(
- { enabled: true, localModelsReady: true, numSpeakers: MAX_SPEAKER_COUNT + 5 },
- null
- );
- assert.equal(above.noteUpdates.expected_speaker_count, MAX_SPEAKER_COUNT);
-
- const unusable = buildUploadNoteMetadata(
- { enabled: true, localModelsReady: true, numSpeakers: 0 },
- null
- );
- assert.equal(unusable.noteUpdates.expected_speaker_count, null);
-});
-
test("unusable durations degrade to null", async () => {
const { buildUploadNoteMetadata } = await load();
const diarization = { enabled: true, localModelsReady: true, numSpeakers: 2 };
@@ -183,7 +166,6 @@ test("segments and diarization metadata share one noteUpdates write", async () =
assert.deepEqual(noteUpdates, {
diarization_enabled: 1,
- expected_speaker_count: 2,
transcript: JSON.stringify([
{ text: "Hi.", timestamp: anchorMs + 1000, speakerName: "Speaker 1" },
]),
@@ -200,21 +182,3 @@ test("upload titles fall back to the transcript, then the file name", async () =
);
assert.equal(uploadTitleFallback(" ", "board-meeting.m4a"), "board-meeting");
});
-
-// uploadNotes.ts carries a renderer twin of the main-process
-// normalizeStoredSpeakerCount (CJS, unloadable from renderer source). Hold the
-// two implementations to identical outputs so they cannot drift apart.
-test("speaker-count normalization matches the main-process implementation", async () => {
- const { buildUploadNoteMetadata } = await load();
- const { normalizeStoredSpeakerCount } = require("../../src/helpers/speakerCount");
-
- const inputs = [1, 2, 3, MAX_SPEAKER_COUNT, MAX_SPEAKER_COUNT + 1, MAX_SPEAKER_COUNT + 5];
- for (const value of [...inputs, 0, -1, 1.5, NaN, Infinity, -Infinity, null, undefined, "", {}]) {
- assert.equal(
- buildUploadNoteMetadata({ enabled: true, numSpeakers: value }).noteUpdates
- .expected_speaker_count,
- normalizeStoredSpeakerCount(value),
- `diverged from normalizeStoredSpeakerCount for ${JSON.stringify(value)}`
- );
- }
-});