Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(transcription): start a speaker only from a 5 s utterance
Elian Doran762a8b18355c
diff --git a/.claude/skills/working-with-transcription/SKILL.md b/.claude/skills/working-with-transcription/SKILL.mdindex 2b979dfa91..75e58b37f7 100644--- a/.claude/skills/working-with-transcription/SKILL.md+++ b/.claude/skills/working-with-transcription/SKILL.md@@ -129,7 +129,7 @@ The app labels the system source's utterances live when `speaker/nemo_en_titanet`diarize.mjs` labels one source's utterances by speaker in two ways and times both:-- **live**: one embedding per utterance, in arrival order, joined to the closest speaker heard so far when the cosine similarity reaches `--threshold` (0.5), else a new speaker; utterances shorter than `--min-duration` (1.5 s) never start one. This is what can run during a meeting.+- **live**: one embedding per utterance, in arrival order, joined to the closest speaker heard so far when the cosine similarity reaches `--threshold` (0.5). Below it, an utterance of at least `--new-min` (5 s) starts a new speaker; a shorter one joins the closest speaker, from `--short-threshold` (0.3) on when it is shorter than `--min-duration` (1.5 s), and only utterances of `--min-duration` or more move a centroid. This is what runs during a meeting (`SPEAKER_SETTINGS`).- **offline**: sherpa-onnx's `OfflineSpeakerDiarization` over the whole source (pyannote segmentation, embeddings, clustering), mapped onto the utterances by overlap; `+Dn` marks an utterance in which a second speaker talks ≥ 1 s.```bashdiff --git a/.claude/skills/working-with-transcription/diarize.mjs b/.claude/skills/working-with-transcription/diarize.mjsindex ba0d64b7d8..7ce8da2478 100644--- a/.claude/skills/working-with-transcription/diarize.mjs+++ b/.claude/skills/working-with-transcription/diarize.mjs@@ -13,7 +13,10 @@// --embedding <name|file> titanet (default), campplus, or a model file// --mode <live|offline|both> (default both)// --threshold <n> live: cosine similarity that joins a known speaker (default 0.5)-// --min-duration <s> live: shorter utterances never start a speaker (default 1.5)+// --min-duration <s> live: shorter utterances never move a centroid (default 1.5)+// --new-min <s> live: only utterances this long start a speaker (default 5)+// --short-threshold <n> live: similarity from which a short utterance joins the nearest+// speaker (default 0.3)// --clusters <n> offline: the number of speakers, when known// --cluster-threshold <n> offline: clustering threshold otherwise (default 0.5)// --threads <n> threads per model (default 1)@@ -105,6 +108,8 @@ if (live && offline) {function labelLive() {const threshold = Number(options.threshold ?? 0.5);const minDuration = Number(options["min-duration"] ?? 1.5);+ const newMin = Number(options["new-min"] ?? 5);+ const shortThreshold = Number(options["short-threshold"] ?? 0.3);const loadStarted = performance.now();const extractor = new sherpa.SpeakerEmbeddingExtractor({model: embeddingModel, numThreads: threads, provider: "cpu"@@ -136,9 +141,11 @@ function labelLive() {addInto(centroids[best], embedding);}labels.push({ speaker: `S${best + 1}`, similarity: bestSimilarity });- } else if (isLongEnough) {+ } else if (utterance.durationMs / 1000 >= newMin) {centroids.push(Float32Array.from(embedding));labels.push({ speaker: `S${centroids.length}`, similarity: best >= 0 ? bestSimilarity : null });+ } else if (best >= 0 && (isLongEnough || bestSimilarity >= shortThreshold)) {+ labels.push({ speaker: `S${best + 1}`, similarity: bestSimilarity });} else {labels.push({ speaker: "?", similarity: best >= 0 ? bestSimilarity : null });}diff --git a/apps/desktop/src/services/transcription/engine.spec.ts b/apps/desktop/src/services/transcription/engine.spec.tsindex 90cb7e121a..6bf3b4c7f6 100644--- a/apps/desktop/src/services/transcription/engine.spec.ts+++ b/apps/desktop/src/services/transcription/engine.spec.ts@@ -184,11 +184,12 @@ describe("loadSherpaEngine", () => {const engine = await loadSherpaEngine(MODELS);expect(h.extractorConfig?.model).toBe(MODELS.speaker);expect(engine.settings.speakers).toMatchObject({- model: "nemo_en_titanet_small.onnx", threshold: 0.5, minDurationSeconds: 1.5+ model: "nemo_en_titanet_small.onnx", threshold: 0.5, minDurationSeconds: 1.5,+ newSpeakerMinDurationSeconds: 5, shortThreshold: 0.3});const tracker = engine.createSpeakerTracker?.();- const long = new Float32Array(16000 * 2);+ const long = new Float32Array(16000 * 6);h.embeddings.push([ 1, 0 ], [ 0, 1 ], [ 0.9, 0.1 ]);expect([ 1, 2, 3 ].map(() => tracker?.identify(long))).toEqual([ 1, 2, 1 ]);diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.tsindex aa262081ff..f56fc20fab 100644--- a/apps/desktop/src/services/transcription/engine.ts+++ b/apps/desktop/src/services/transcription/engine.ts@@ -68,7 +68,9 @@ const DECODING_SETTINGS: Record<ModelPrecision, Record<string, unknown>> = {const SPEAKER_SETTINGS: SpeakerClustererOptions = {threshold: 0.5,- minDurationSeconds: 1.5+ minDurationSeconds: 1.5,+ newSpeakerMinDurationSeconds: 5,+ shortThreshold: 0.3};const VAD_SETTINGS = {diff --git a/apps/desktop/src/services/transcription/speakers.spec.ts b/apps/desktop/src/services/transcription/speakers.spec.tsindex ce4049ddb4..474985353e 100644--- a/apps/desktop/src/services/transcription/speakers.spec.ts+++ b/apps/desktop/src/services/transcription/speakers.spec.ts@@ -2,43 +2,65 @@ import { describe, expect, it } from "vitest";import { SpeakerClusterer } from "./speakers";-const options = { threshold: 0.5, minDurationSeconds: 1.5 };+const options = {+ threshold: 0.5,+ minDurationSeconds: 1.5,+ newSpeakerMinDurationSeconds: 5,+ shortThreshold: 0.3+};describe("SpeakerClusterer", () => {it("joins similar embeddings and starts a speaker for a distant one", () => {const clusterer = new SpeakerClusterer(options);- expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 3)).toBe(1);+ expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 6)).toBe(1);// Scale does not matter, only direction.- expect(clusterer.assign(new Float32Array([ 9, 1, 0 ]), 3)).toBe(1);- expect(clusterer.assign(new Float32Array([ 0, 1, 0 ]), 3)).toBe(2);- expect(clusterer.assign(new Float32Array([ 0.1, 1, 0 ]), 3)).toBe(2);- expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 3)).toBe(3);+ expect(clusterer.assign(new Float32Array([ 9, 1, 0 ]), 6)).toBe(1);+ expect(clusterer.assign(new Float32Array([ 0, 1, 0 ]), 6)).toBe(2);+ expect(clusterer.assign(new Float32Array([ 0.1, 1, 0 ]), 6)).toBe(2);+ expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 6)).toBe(3);});it("takes a silent, all-zero embedding as a speaker of its own rather than dividing by zero", () => {const clusterer = new SpeakerClusterer(options);- expect(clusterer.assign(new Float32Array([ 0, 0, 0 ]), 3)).toBe(1);- expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 3)).toBe(2);+ expect(clusterer.assign(new Float32Array([ 0, 0, 0 ]), 6)).toBe(1);+ expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 6)).toBe(2);});it("lets a short utterance join a speaker but never start one or move a centroid", () => {const clusterer = new SpeakerClusterer(options);expect(clusterer.assign(new Float32Array([ 0, 1 ]), 1)).toBeNull();- expect(clusterer.assign(new Float32Array([ 1, 0 ]), 3)).toBe(1);+ expect(clusterer.assign(new Float32Array([ 1, 0 ]), 6)).toBe(1);// Cosine 0.71 joins speaker 1; a centroid moved by it would let [0, 1] join too.expect(clusterer.assign(new Float32Array([ 1, 1 ]), 1)).toBe(1);expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 1)).toBeNull();- expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 2)).toBe(2);+ expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 6)).toBe(2);+ });++ it("gives an utterance too short to start a speaker the nearest one, without moving it", () => {+ const clusterer = new SpeakerClusterer(options);+ expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 3)).toBeNull();+ clusterer.assign(new Float32Array([ 1, 0, 0 ]), 6);+ clusterer.assign(new Float32Array([ 0, 1, 0 ]), 6);++ // Cosine 0.37 with speaker 1 and 0.18 with speaker 2: under the threshold, nearest to 1.+ expect(clusterer.assign(new Float32Array([ 0.4, 0.2, 1 ]), 3)).toBe(1);+ // A short utterance takes it from the short threshold, 0.3, on.+ expect(clusterer.assign(new Float32Array([ 0.4, 0.2, 1 ]), 1)).toBe(1);+ // Cosine 0.20: too far for a short utterance, but not for a 3 s one.+ expect(clusterer.assign(new Float32Array([ 0.2, 0.1, 1 ]), 1)).toBeNull();+ expect(clusterer.assign(new Float32Array([ 0.2, 0.1, 1 ]), 3)).toBe(1);+ // Cosine 0.77 with a speaker 1 moved by the joins above; 0 with the unmoved one.+ expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 6)).toBe(3);});it("moves a centroid with every long utterance it takes", () => {const clusterer = new SpeakerClusterer(options);- clusterer.assign(new Float32Array([ 1, 0 ]), 3);+ clusterer.assign(new Float32Array([ 1, 0 ]), 6);// Cosine 0.6 with [1, 0] joins, and pulls the centroid toward [0.6, 0.8].expect(clusterer.assign(new Float32Array([ 0.6, 0.8 ]), 3)).toBe(1);// Cosine 0.28 with [1, 0] alone; 0.68 with the moved centroid.- expect(clusterer.assign(new Float32Array([ 0.28, 0.96 ]), 3)).toBe(1);+ expect(clusterer.assign(new Float32Array([ 0.28, 0.96 ]), 6)).toBe(1);});});diff --git a/apps/desktop/src/services/transcription/speakers.ts b/apps/desktop/src/services/transcription/speakers.tsindex d0ead044bc..f4b2d5e393 100644--- a/apps/desktop/src/services/transcription/speakers.ts+++ b/apps/desktop/src/services/transcription/speakers.ts@@ -1,14 +1,22 @@export interface SpeakerClustererOptions {/** The cosine similarity to a speaker's centroid from which an utterance joins that speaker. */threshold: number;- /** Utterances shorter than this, in seconds, never start a speaker nor move a centroid. */+ /** Utterances shorter than this, in seconds, never move a centroid. */minDurationSeconds: number;+ /** Only an utterance at least this long, in seconds, starts a speaker. */+ newSpeakerMinDurationSeconds: number;+ /**+ * The similarity from which an utterance shorter than `minDurationSeconds` joins the nearest+ * speaker when it is under `threshold`.+ */+ shortThreshold: number;}/*** Assigns embeddings to speakers as they arrive: an embedding joins the speaker whose centroid it- * is most similar to, when that similarity reaches the threshold, and starts a new speaker- * otherwise.+ * is most similar to, when that similarity reaches the threshold. Below it, an utterance of at+ * least `newSpeakerMinDurationSeconds` starts a new speaker, and a shorter one joins the nearest+ * speaker without moving its centroid, so a short or noisy utterance never adds a speaker.*/export class SpeakerClusterer {/** The sum of each speaker's normalized embeddings; its direction is the centroid. */@@ -35,11 +43,14 @@ export class SpeakerClusterer {}return best + 1;}- if (!isLongEnough) {- return null;+ if (durationSeconds >= this.options.newSpeakerMinDurationSeconds) {+ this.centroids.push(normalized);+ return this.centroids.length;}- this.centroids.push(normalized);- return this.centroids.length;+ if (best >= 0 && (isLongEnough || bestSimilarity >= this.options.shortThreshold)) {+ return best + 1;+ }+ return null;}}