Skip to content

Build your personal knowledge base with Trilium Notes

feat(transcription): start a speaker only from a 5 s utterance

Elian Doran762a8b18355c
diff --git a/.claude/skills/working-with-transcription/SKILL.md b/.claude/skills/working-with-transcription/SKILL.md
index 2b979dfa91..75e58b37f7 100644
--- a/.claude/skills/working-with-transcription/SKILL.md
+++ b/.claude/skills/working-with-transcription/SKILL.md
@@ -129,7 +129,7 @@ The app labels the system source's utterances live when `speaker/nemo_en_titanet
`diarize.mjs` labels one source's utterances by speaker in two ways and times both:
-- **live**: one embedding per utterance, in arrival order, joined to the closest speaker heard so far when the cosine similarity reaches `--threshold` (0.5), else a new speaker; utterances shorter than `--min-duration` (1.5 s) never start one. This is what can run during a meeting.
+- **live**: one embedding per utterance, in arrival order, joined to the closest speaker heard so far when the cosine similarity reaches `--threshold` (0.5). Below it, an utterance of at least `--new-min` (5 s) starts a new speaker; a shorter one joins the closest speaker, from `--short-threshold` (0.3) on when it is shorter than `--min-duration` (1.5 s), and only utterances of `--min-duration` or more move a centroid. This is what runs during a meeting (`SPEAKER_SETTINGS`).
- **offline**: sherpa-onnx's `OfflineSpeakerDiarization` over the whole source (pyannote segmentation, embeddings, clustering), mapped onto the utterances by overlap; `+Dn` marks an utterance in which a second speaker talks ≥ 1 s.
```bash
diff --git a/.claude/skills/working-with-transcription/diarize.mjs b/.claude/skills/working-with-transcription/diarize.mjs
index ba0d64b7d8..7ce8da2478 100644
--- a/.claude/skills/working-with-transcription/diarize.mjs
+++ b/.claude/skills/working-with-transcription/diarize.mjs
@@ -13,7 +13,10 @@
// --embedding <name|file> titanet (default), campplus, or a model file
// --mode <live|offline|both> (default both)
// --threshold <n> live: cosine similarity that joins a known speaker (default 0.5)
-// --min-duration <s> live: shorter utterances never start a speaker (default 1.5)
+// --min-duration <s> live: shorter utterances never move a centroid (default 1.5)
+// --new-min <s> live: only utterances this long start a speaker (default 5)
+// --short-threshold <n> live: similarity from which a short utterance joins the nearest
+// speaker (default 0.3)
// --clusters <n> offline: the number of speakers, when known
// --cluster-threshold <n> offline: clustering threshold otherwise (default 0.5)
// --threads <n> threads per model (default 1)
@@ -105,6 +108,8 @@ if (live && offline) {
function labelLive() {
const threshold = Number(options.threshold ?? 0.5);
const minDuration = Number(options["min-duration"] ?? 1.5);
+ const newMin = Number(options["new-min"] ?? 5);
+ const shortThreshold = Number(options["short-threshold"] ?? 0.3);
const loadStarted = performance.now();
const extractor = new sherpa.SpeakerEmbeddingExtractor({
model: embeddingModel, numThreads: threads, provider: "cpu"
@@ -136,9 +141,11 @@ function labelLive() {
addInto(centroids[best], embedding);
}
labels.push({ speaker: `S${best + 1}`, similarity: bestSimilarity });
- } else if (isLongEnough) {
+ } else if (utterance.durationMs / 1000 >= newMin) {
centroids.push(Float32Array.from(embedding));
labels.push({ speaker: `S${centroids.length}`, similarity: best >= 0 ? bestSimilarity : null });
+ } else if (best >= 0 && (isLongEnough || bestSimilarity >= shortThreshold)) {
+ labels.push({ speaker: `S${best + 1}`, similarity: bestSimilarity });
} else {
labels.push({ speaker: "?", similarity: best >= 0 ? bestSimilarity : null });
}
diff --git a/apps/desktop/src/services/transcription/engine.spec.ts b/apps/desktop/src/services/transcription/engine.spec.ts
index 90cb7e121a..6bf3b4c7f6 100644
--- a/apps/desktop/src/services/transcription/engine.spec.ts
+++ b/apps/desktop/src/services/transcription/engine.spec.ts
@@ -184,11 +184,12 @@ describe("loadSherpaEngine", () => {
const engine = await loadSherpaEngine(MODELS);
expect(h.extractorConfig?.model).toBe(MODELS.speaker);
expect(engine.settings.speakers).toMatchObject({
- model: "nemo_en_titanet_small.onnx", threshold: 0.5, minDurationSeconds: 1.5
+ model: "nemo_en_titanet_small.onnx", threshold: 0.5, minDurationSeconds: 1.5,
+ newSpeakerMinDurationSeconds: 5, shortThreshold: 0.3
});
const tracker = engine.createSpeakerTracker?.();
- const long = new Float32Array(16000 * 2);
+ const long = new Float32Array(16000 * 6);
h.embeddings.push([ 1, 0 ], [ 0, 1 ], [ 0.9, 0.1 ]);
expect([ 1, 2, 3 ].map(() => tracker?.identify(long))).toEqual([ 1, 2, 1 ]);
diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.ts
index aa262081ff..f56fc20fab 100644
--- a/apps/desktop/src/services/transcription/engine.ts
+++ b/apps/desktop/src/services/transcription/engine.ts
@@ -68,7 +68,9 @@ const DECODING_SETTINGS: Record<ModelPrecision, Record<string, unknown>> = {
const SPEAKER_SETTINGS: SpeakerClustererOptions = {
threshold: 0.5,
- minDurationSeconds: 1.5
+ minDurationSeconds: 1.5,
+ newSpeakerMinDurationSeconds: 5,
+ shortThreshold: 0.3
};
const VAD_SETTINGS = {
diff --git a/apps/desktop/src/services/transcription/speakers.spec.ts b/apps/desktop/src/services/transcription/speakers.spec.ts
index ce4049ddb4..474985353e 100644
--- a/apps/desktop/src/services/transcription/speakers.spec.ts
+++ b/apps/desktop/src/services/transcription/speakers.spec.ts
@@ -2,43 +2,65 @@ import { describe, expect, it } from "vitest";
import { SpeakerClusterer } from "./speakers";
-const options = { threshold: 0.5, minDurationSeconds: 1.5 };
+const options = {
+ threshold: 0.5,
+ minDurationSeconds: 1.5,
+ newSpeakerMinDurationSeconds: 5,
+ shortThreshold: 0.3
+};
describe("SpeakerClusterer", () => {
it("joins similar embeddings and starts a speaker for a distant one", () => {
const clusterer = new SpeakerClusterer(options);
- expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 3)).toBe(1);
+ expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 6)).toBe(1);
// Scale does not matter, only direction.
- expect(clusterer.assign(new Float32Array([ 9, 1, 0 ]), 3)).toBe(1);
- expect(clusterer.assign(new Float32Array([ 0, 1, 0 ]), 3)).toBe(2);
- expect(clusterer.assign(new Float32Array([ 0.1, 1, 0 ]), 3)).toBe(2);
- expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 3)).toBe(3);
+ expect(clusterer.assign(new Float32Array([ 9, 1, 0 ]), 6)).toBe(1);
+ expect(clusterer.assign(new Float32Array([ 0, 1, 0 ]), 6)).toBe(2);
+ expect(clusterer.assign(new Float32Array([ 0.1, 1, 0 ]), 6)).toBe(2);
+ expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 6)).toBe(3);
});
it("takes a silent, all-zero embedding as a speaker of its own rather than dividing by zero", () => {
const clusterer = new SpeakerClusterer(options);
- expect(clusterer.assign(new Float32Array([ 0, 0, 0 ]), 3)).toBe(1);
- expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 3)).toBe(2);
+ expect(clusterer.assign(new Float32Array([ 0, 0, 0 ]), 6)).toBe(1);
+ expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 6)).toBe(2);
});
it("lets a short utterance join a speaker but never start one or move a centroid", () => {
const clusterer = new SpeakerClusterer(options);
expect(clusterer.assign(new Float32Array([ 0, 1 ]), 1)).toBeNull();
- expect(clusterer.assign(new Float32Array([ 1, 0 ]), 3)).toBe(1);
+ expect(clusterer.assign(new Float32Array([ 1, 0 ]), 6)).toBe(1);
// Cosine 0.71 joins speaker 1; a centroid moved by it would let [0, 1] join too.
expect(clusterer.assign(new Float32Array([ 1, 1 ]), 1)).toBe(1);
expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 1)).toBeNull();
- expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 2)).toBe(2);
+ expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 6)).toBe(2);
+ });
+
+ it("gives an utterance too short to start a speaker the nearest one, without moving it", () => {
+ const clusterer = new SpeakerClusterer(options);
+ expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 3)).toBeNull();
+ clusterer.assign(new Float32Array([ 1, 0, 0 ]), 6);
+ clusterer.assign(new Float32Array([ 0, 1, 0 ]), 6);
+
+ // Cosine 0.37 with speaker 1 and 0.18 with speaker 2: under the threshold, nearest to 1.
+ expect(clusterer.assign(new Float32Array([ 0.4, 0.2, 1 ]), 3)).toBe(1);
+ // A short utterance takes it from the short threshold, 0.3, on.
+ expect(clusterer.assign(new Float32Array([ 0.4, 0.2, 1 ]), 1)).toBe(1);
+ // Cosine 0.20: too far for a short utterance, but not for a 3 s one.
+ expect(clusterer.assign(new Float32Array([ 0.2, 0.1, 1 ]), 1)).toBeNull();
+ expect(clusterer.assign(new Float32Array([ 0.2, 0.1, 1 ]), 3)).toBe(1);
+ // Cosine 0.77 with a speaker 1 moved by the joins above; 0 with the unmoved one.
+ expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 6)).toBe(3);
});
it("moves a centroid with every long utterance it takes", () => {
const clusterer = new SpeakerClusterer(options);
- clusterer.assign(new Float32Array([ 1, 0 ]), 3);
+ clusterer.assign(new Float32Array([ 1, 0 ]), 6);
// Cosine 0.6 with [1, 0] joins, and pulls the centroid toward [0.6, 0.8].
expect(clusterer.assign(new Float32Array([ 0.6, 0.8 ]), 3)).toBe(1);
// Cosine 0.28 with [1, 0] alone; 0.68 with the moved centroid.
- expect(clusterer.assign(new Float32Array([ 0.28, 0.96 ]), 3)).toBe(1);
+ expect(clusterer.assign(new Float32Array([ 0.28, 0.96 ]), 6)).toBe(1);
});
});
diff --git a/apps/desktop/src/services/transcription/speakers.ts b/apps/desktop/src/services/transcription/speakers.ts
index d0ead044bc..f4b2d5e393 100644
--- a/apps/desktop/src/services/transcription/speakers.ts
+++ b/apps/desktop/src/services/transcription/speakers.ts
@@ -1,14 +1,22 @@
export interface SpeakerClustererOptions {
/** The cosine similarity to a speaker's centroid from which an utterance joins that speaker. */
threshold: number;
- /** Utterances shorter than this, in seconds, never start a speaker nor move a centroid. */
+ /** Utterances shorter than this, in seconds, never move a centroid. */
minDurationSeconds: number;
+ /** Only an utterance at least this long, in seconds, starts a speaker. */
+ newSpeakerMinDurationSeconds: number;
+ /**
+ * The similarity from which an utterance shorter than `minDurationSeconds` joins the nearest
+ * speaker when it is under `threshold`.
+ */
+ shortThreshold: number;
}
/**
* Assigns embeddings to speakers as they arrive: an embedding joins the speaker whose centroid it
- * is most similar to, when that similarity reaches the threshold, and starts a new speaker
- * otherwise.
+ * is most similar to, when that similarity reaches the threshold. Below it, an utterance of at
+ * least `newSpeakerMinDurationSeconds` starts a new speaker, and a shorter one joins the nearest
+ * speaker without moving its centroid, so a short or noisy utterance never adds a speaker.
*/
export class SpeakerClusterer {
/** The sum of each speaker's normalized embeddings; its direction is the centroid. */
@@ -35,11 +43,14 @@ export class SpeakerClusterer {
}
return best + 1;
}
- if (!isLongEnough) {
- return null;
+ if (durationSeconds >= this.options.newSpeakerMinDurationSeconds) {
+ this.centroids.push(normalized);
+ return this.centroids.length;
}
- this.centroids.push(normalized);
- return this.centroids.length;
+ if (best >= 0 && (isLongEnough || bestSimilarity >= this.options.shortThreshold)) {
+ return best + 1;
+ }
+ return null;
}
}