Skip to content

Build your personal knowledge base with Trilium Notes

feat(desktop): tell the speakers on the system audio apart

Elian Doraneda0f3c5a263
diff --git a/.claude/skills/working-with-transcription/SKILL.md b/.claude/skills/working-with-transcription/SKILL.md
index 2ca7369a33..0ec68b0ae1 100644
--- a/.claude/skills/working-with-transcription/SKILL.md
+++ b/.claude/skills/working-with-transcription/SKILL.md
@@ -90,7 +90,9 @@ To measure accuracy rather than difference, make a reference: copy the baseline
## Speaker detection
-Not in the app yet. `diarize.mjs` labels one source's utterances by speaker in two ways and times both:
+The app labels the system source's utterances live when `speaker/nemo_en_titanet_small.onnx` is installed: `EmbeddingSpeakerTracker` in `engine.ts` embeds each transcribed utterance and `SpeakerClusterer` (`speakers.ts`, settings `SPEAKER_SETTINGS`) assigns it, giving the utterance a `speaker` number that `TranscriptView` shows as "Speaker n". The microphone is never labelled. It runs synchronously on Electron's main thread for now, ~100 ms per utterance. `replay.mjs` does not label speakers; use `diarize.mjs`.
+
+`diarize.mjs` labels one source's utterances by speaker in two ways and times both:
- **live**: one embedding per utterance, in arrival order, joined to the closest speaker heard so far when the cosine similarity reaches `--threshold` (0.5), else a new speaker; utterances shorter than `--min-duration` (1.5 s) never start one. This is what can run during a meeting.
- **offline**: sherpa-onnx's `OfflineSpeakerDiarization` over the whole source (pyannote segmentation, embeddings, clustering), mapped onto the utterances by overlap; `+Dn` marks an utterance in which a second speaker talks ≥ 1 s.
@@ -106,7 +108,7 @@ Models, in `<data dir>/transcription-model/speaker/`, from `k2-fsa/sherpa-onnx`
- `nemo_en_titanet_small.onnx` (40 MB) and `3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx` (30 MB) from `speaker-recongition-models` (the tag is misspelled upstream);
- `segmentation.onnx` (6 MB): `model.onnx` of `sherpa-onnx-pyannote-segmentation-3-0.tar.bz2` from `speaker-segmentation-models`, renamed.
-`SpeakerEmbeddingExtractor.compute()` and `OfflineSpeakerDiarization.process()` are **synchronous**, with no async variant: in the app they must not run on Electron's main thread (a worker thread, or the capture window).
+`SpeakerEmbeddingExtractor.compute()` and `OfflineSpeakerDiarization.process()` are **synchronous**, with no async variant. Live labelling blocks Electron's main thread, which also serves the backend, for ~100 ms per system utterance; the planned fix is to move the whole pipeline (VAD, speakers, recognizer) into one worker thread. The offline method, ~20 s per two minutes, could never run there.
Results on the first recording (two minutes, interviewer and curator on the system track, one thread):
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json
index 8fc95f799e..d5ca354453 100644
--- a/apps/client/src/translations/en/translation.json
+++ b/apps/client/src/translations/en/translation.json
@@ -4588,6 +4588,7 @@
"transcript": {
"speaker_me": "Me",
"speaker_others": "Others",
+ "speaker_numbered": "Speaker {{number}}",
"empty": "Nothing has been transcribed yet.",
"unreadable": "This transcript cannot be read.",
"speaking": "Speaking…",
diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx
index 6a70831198..9dc129ef1f 100644
--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx
+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx
@@ -54,6 +54,23 @@ describe("transcript helpers", () => {
]);
});
+ it("starts a turn when the speaker of a source changes, not for an unidentified one", () => {
+ const turns = groupIntoTurns([
+ { source: "system", offsetMs: 0, text: "Hi." },
+ { source: "system", offsetMs: 1000, speaker: 1, text: "Welcome to the show." },
+ { source: "system", offsetMs: 4000, text: "Yes." },
+ { source: "system", offsetMs: 5000, speaker: 1, text: "Our guest is here." },
+ { source: "system", offsetMs: 8000, speaker: 2, text: "Thanks for having me." }
+ ]);
+ expect(turns.map(({ speaker, offsetMs }) => ({ speaker, offsetMs }))).toEqual([
+ { speaker: 1, offsetMs: 0 },
+ { speaker: 2, offsetMs: 8000 }
+ ]);
+ expect(turns.map(paragraphTexts)).toEqual([
+ [ "Hi. Welcome to the show. Yes. Our guest is here." ], [ "Thanks for having me." ]
+ ]);
+ });
+
it("starts a new paragraph within a turn after a long pause", () => {
const turns = groupIntoTurns([
{ source: "system", offsetMs: 0, durationMs: 8000, text: "First point," },
@@ -81,7 +98,7 @@ describe("TranscriptView", () => {
.mockResolvedValueOnce({ content: transcript([
{ source: "microphone", offsetMs: 1500, durationMs: 1000, text: "Can you hear me?" },
{ source: "microphone", offsetMs: 6000, durationMs: 1000, text: "Hello?" },
- { source: "system", offsetMs: 8000, text: "Yes." }
+ { source: "system", offsetMs: 8000, speaker: 2, text: "Yes." }
]) });
const attachment = { attachmentId: "transcript1", getBlob } as unknown as FAttachment;
const component = new Component();
@@ -104,7 +121,7 @@ describe("TranscriptView", () => {
await vi.waitFor(() => expect(turns()).toHaveLength(2));
expect(turns()).toEqual([
"transcript.speaker_me14:29Can you hear me?Hello?",
- "transcript.speaker_others14:30Yes."
+ "transcript.speaker_numbered14:30Yes."
]);
expect(container.querySelectorAll(".transcript-microphone p")).toHaveLength(2);
});
diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx
index 3c0da7cfe2..8bfa312881 100644
--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx
+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx
@@ -89,7 +89,7 @@ export default function TranscriptView({ attachment }: { attachment: FAttachment
{turns.map((turn) => (
<div key={turn.offsetMs} className={`transcript-turn transcript-${turn.source}`}>
<div className="transcript-turn-header">
- <span className="transcript-speaker">{speakerName(turn.source)}</span>
+ <span className="transcript-speaker">{speakerName(turn.source, turn.speaker)}</span>
<TurnTime startedAt={transcript.startedAt} offsetMs={turn.offsetMs} />
</div>
{turn.paragraphs.map((paragraph, index) => (
@@ -142,9 +142,11 @@ function TurnTime({ startedAt, offsetMs }: { startedAt: string; offsetMs: number
);
}
-/** Consecutive utterances from one source, split into paragraphs at long pauses. */
+/** Consecutive utterances of one speaker, split into paragraphs at long pauses. */
export interface TranscriptTurn {
source: TranscriptSource;
+ /** The first speaker identified in the turn, if any. */
+ speaker?: number;
offsetMs: number;
paragraphs: TranscriptUtterance[][];
}
@@ -165,22 +167,28 @@ export function parseTranscript(content: string | null): Transcript | null {
}
/**
- * Joins each run of utterances from one source into a turn, timed by its first utterance, and
- * starts a new paragraph after a pause of {@link PARAGRAPH_PAUSE_MS} or more. An utterance without
- * a `durationMs` never starts one, since the pause before the next is unknown.
+ * Joins each run of utterances from one source and speaker into a turn, timed by its first
+ * utterance, and starts a new paragraph after a pause of {@link PARAGRAPH_PAUSE_MS} or more. An
+ * utterance without a speaker continues the turn of its source, and an utterance without a
+ * `durationMs` never starts a paragraph, since the pause before the next is unknown.
*/
export function groupIntoTurns(utterances: TranscriptUtterance[]): TranscriptTurn[] {
const turns: TranscriptTurn[] = [];
let previousEndMs: number | null = null;
for (const utterance of utterances) {
- const { source, offsetMs, durationMs } = utterance;
+ const { source, speaker, offsetMs, durationMs } = utterance;
const last = turns.at(-1);
- if (last?.source !== source) {
- turns.push({ source, offsetMs, paragraphs: [ [ utterance ] ] });
- } else if (previousEndMs !== null && offsetMs - previousEndMs >= PARAGRAPH_PAUSE_MS) {
- last.paragraphs.push([ utterance ]);
+ const isSameSpeaker = speaker === undefined || last?.speaker === undefined
+ || last.speaker === speaker;
+ if (last?.source !== source || !isSameSpeaker) {
+ turns.push({ source, speaker, offsetMs, paragraphs: [ [ utterance ] ] });
} else {
- last.paragraphs.at(-1)?.push(utterance);
+ last.speaker ??= speaker;
+ if (previousEndMs !== null && offsetMs - previousEndMs >= PARAGRAPH_PAUSE_MS) {
+ last.paragraphs.push([ utterance ]);
+ } else {
+ last.paragraphs.at(-1)?.push(utterance);
+ }
}
previousEndMs = durationMs === undefined ? null : offsetMs + durationMs;
}
@@ -203,6 +211,11 @@ function wasAtEnd(scroller: HTMLElement, scrollHeight: number) {
return scrollHeight - scroller.scrollTop - scroller.clientHeight <= FOLLOW_THRESHOLD;
}
-function speakerName(source: TranscriptSource) {
- return source === "microphone" ? t("transcript.speaker_me") : t("transcript.speaker_others");
+function speakerName(source: TranscriptSource, speaker?: number) {
+ if (source === "microphone") {
+ return t("transcript.speaker_me");
+ }
+ return speaker === undefined
+ ? t("transcript.speaker_others")
+ : t("transcript.speaker_numbered", { number: speaker });
}
diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.ts
index 0eca27eda7..9c573168dd 100644
--- a/apps/desktop/src/services/transcription/engine.ts
+++ b/apps/desktop/src/services/transcription/engine.ts
@@ -1,7 +1,9 @@
import dataDirs from "@triliumnext/server/src/services/data_dir.js";
import fs from "fs";
import path from "path";
-import type { Vad } from "sherpa-onnx-node";
+import type { SpeakerEmbeddingExtractor, Vad } from "sherpa-onnx-node";
+
+import { SpeakerClusterer, type SpeakerClustererOptions } from "./speakers";
/** Every source and engine works on mono 16 kHz samples. */
export const SAMPLE_RATE = 16000;
@@ -27,9 +29,20 @@ export interface SpeechSegmenter {
isSpeaking(): boolean;
}
+/** Tells the speakers of one source apart. Each source needs its own. */
+export interface SpeakerTracker {
+ /**
+ * Returns the speaker of an utterance, numbered from 1 in the order they were first heard, or
+ * `null` when the utterance is too short to tell.
+ */
+ identify(samples: Float32Array): number | null;
+}
+
export interface SpeechEngine {
recognizer: SpeechRecognizer;
createSegmenter(): SpeechSegmenter;
+ /** Absent when the speaker model is not installed. */
+ createSpeakerTracker?(): SpeakerTracker;
/** The settings that shape the transcript, kept with a recording to compare engines by. */
settings: Record<string, unknown>;
}
@@ -37,6 +50,9 @@ export interface SpeechEngine {
/** Where the models live: Parakeet TDT 0.6B v3 (int8) plus `silero_vad.onnx`. */
export const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");
+/** The speaker embedding model, optional: without it, utterances carry no speaker. */
+const SPEAKER_MODEL = path.join(MODEL_DIR, "speaker", "nemo_en_titanet_small.onnx");
+
/** Silero VAD consumes 512-sample windows at 16 kHz. */
const VAD_WINDOW_SIZE = 512;
@@ -48,6 +64,11 @@ const RECOGNIZER_SETTINGS = {
const DECODING_METHOD = "greedy_search";
+const SPEAKER_SETTINGS: SpeakerClustererOptions = {
+ threshold: 0.5,
+ minDurationSeconds: 1.5
+};
+
const VAD_SETTINGS = {
threshold: 0.5,
minSilenceDuration: 0.5,
@@ -83,6 +104,12 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {
decodingMethod: DECODING_METHOD
});
+ const speakerExtractor = fs.existsSync(SPEAKER_MODEL)
+ ? new sherpa.SpeakerEmbeddingExtractor({
+ model: SPEAKER_MODEL, numThreads: 1, provider: "cpu"
+ })
+ : null;
+
return {
recognizer: {
async transcribe(samples) {
@@ -100,10 +127,16 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {
numThreads: 1,
provider: "cpu"
}, 60)),
+ createSpeakerTracker: speakerExtractor
+ ? () => new EmbeddingSpeakerTracker(speakerExtractor, SPEAKER_SETTINGS)
+ : undefined,
settings: {
modelDirectory: path.resolve(MODEL_DIR),
recognizer: { ...RECOGNIZER_SETTINGS, decodingMethod: DECODING_METHOD },
- vad: VAD_SETTINGS
+ vad: VAD_SETTINGS,
+ speakers: speakerExtractor
+ ? { model: path.basename(SPEAKER_MODEL), ...SPEAKER_SETTINGS }
+ : null
}
};
}
@@ -143,3 +176,30 @@ class VadSegmenter implements SpeechSegmenter {
return segments;
}
}
+
+/**
+ * Identifies speakers by clustering TitaNet embeddings. The embedding is computed synchronously,
+ * about 100 ms per utterance on one thread.
+ */
+class EmbeddingSpeakerTracker implements SpeakerTracker {
+ private readonly clusterer: SpeakerClusterer;
+
+ constructor(
+ private readonly extractor: SpeakerEmbeddingExtractor,
+ options: SpeakerClustererOptions
+ ) {
+ this.clusterer = new SpeakerClusterer(options);
+ }
+
+ identify(samples: Float32Array) {
+ const stream = this.extractor.createStream();
+ stream.acceptWaveform({ samples, sampleRate: SAMPLE_RATE });
+ stream.inputFinished();
+ if (!this.extractor.isReady(stream)) {
+ return null;
+ }
+ // Electron's V8 memory cage rejects the external buffers sherpa-onnx uses by default.
+ const embedding = this.extractor.compute(stream, false);
+ return this.clusterer.assign(embedding, samples.length / SAMPLE_RATE);
+ }
+}
diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.ts
index b6292b5db6..7b5545f74c 100644
--- a/apps/desktop/src/services/transcription/session.spec.ts
+++ b/apps/desktop/src/services/transcription/session.spec.ts
@@ -2,7 +2,7 @@ import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { SamplesListener } from "./capture";
-import type { SpeechSegment } from "./engine";
+import type { SpeakerTracker, SpeechSegment } from "./engine";
import { type AudioTap, RecordingSession, type TranscriptSink } from "./session";
const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));
@@ -49,6 +49,7 @@ function createHarness(extraSinks: TranscriptSink[] = [], taps: AudioTap[] = [])
transcribe: vi.fn(async (samples: Float32Array) =>
samples[0] ? `utterance ${samples[0]}` : "")
},
+ createSpeakerTracker: undefined as (() => SpeakerTracker) | undefined,
createSegmenter: vi.fn(() => {
const pending = flushed.shift() ?? [];
let speaking = false;
@@ -206,6 +207,36 @@ describe("RecordingSession", () => {
expect(capture.start).not.toHaveBeenCalled();
});
+ it("labels the system source's utterances by speaker, never the microphone's", async () => {
+ const { session, engine, deliver, written } = createHarness();
+ const identify = vi.fn()
+ .mockReturnValueOnce(2)
+ .mockImplementationOnce(() => {
+ throw new Error("model crashed");
+ });
+ const createSpeakerTracker = vi.fn(() => ({ identify }));
+ engine.createSpeakerTracker = createSpeakerTracker;
+ await session.start();
+
+ deliver("system", utteranceChunk(1, 0), 2000);
+ deliver("microphone", utteranceChunk(2, 0), 2000);
+ deliver("system", utteranceChunk(3, 16000), 3000);
+ // Transcribes to nothing, so its speaker is never asked for.
+ deliver("system", utteranceChunk(0, 32000), 4000);
+ await session.stop();
+
+ expect(createSpeakerTracker).toHaveBeenCalledOnce();
+ expect(identify).toHaveBeenCalledTimes(2);
+ expect(written.map(({ source, speaker, text }) => ({ source, speaker, text }))).toEqual([
+ { source: "system", speaker: 2, text: "utterance 1" },
+ { source: "microphone", speaker: undefined, text: "utterance 2" },
+ { source: "system", speaker: undefined, text: "utterance 3" }
+ ]);
+ expect(written[1]).not.toHaveProperty("speaker");
+ expect(log.error).toHaveBeenCalledWith(
+ "Transcription: identifying the speaker on system failed: Error: model crashed");
+ });
+
it("logs a failing source instead of throwing into the IPC listener", async () => {
const { session, engine, deliver } = createHarness();
engine.createSegmenter.mockImplementationOnce(() => {
diff --git a/apps/desktop/src/services/transcription/session.ts b/apps/desktop/src/services/transcription/session.ts
index 9645f793f0..e733b6c9b2 100644
--- a/apps/desktop/src/services/transcription/session.ts
+++ b/apps/desktop/src/services/transcription/session.ts
@@ -2,7 +2,12 @@ import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons
import { getLog } from "@triliumnext/core";
import type { AudioCapture } from "./capture";
-import { SAMPLE_RATE, type SpeechEngine, type SpeechSegment, type SpeechSegmenter } from "./engine";
+import {
+ SAMPLE_RATE, type SpeakerTracker, type SpeechEngine, type SpeechSegment, type SpeechSegmenter
+} from "./engine";
+
+/** The sources whose speakers are told apart; the microphone carries only the local speaker. */
+const DIARIZED_SOURCES: readonly TranscriptSource[] = [ "system" ];
/** Receives the utterances of a recording in the order they were spoken per source. */
export interface TranscriptSink {
@@ -49,6 +54,7 @@ export interface RecordingSessionOptions {
*/
export class RecordingSession {
private readonly pipelines = new Map<TranscriptSource, SourcePipeline>();
+ private readonly speakerTrackers = new Map<TranscriptSource, SpeakerTracker>();
/** How many utterances of each source wait to be transcribed or written. */
private readonly queued = new Map<TranscriptSource, number>();
private queue = Promise.resolve();
@@ -122,6 +128,12 @@ export class RecordingSession {
(offsetMs, samples) => this.enqueue(source, offsetMs, samples)
);
this.pipelines.set(source, pipeline);
+ const tracker = DIARIZED_SOURCES.includes(source)
+ ? this.options.engine.createSpeakerTracker?.()
+ : undefined;
+ if (tracker) {
+ this.speakerTrackers.set(source, tracker);
+ }
}
return pipeline;
}
@@ -135,9 +147,13 @@ export class RecordingSession {
}
const durationMs = samples.length / SAMPLE_RATE * 1000;
+ const speaker = this.identifySpeaker(source, samples);
+ const utterance = {
+ source, offsetMs, durationMs, text, ...(speaker === null ? {} : { speaker })
+ };
for (const sink of this.options.sinks) {
try {
- await sink.write({ source, offsetMs, durationMs, text });
+ await sink.write(utterance);
} catch (e) {
getLog().error(`Transcription: a sink failed: ${e}`);
}
@@ -148,6 +164,16 @@ export class RecordingSession {
});
}
+ /** Runs on the main thread, synchronously: about 100 ms per utterance. */
+ private identifySpeaker(source: TranscriptSource, samples: Float32Array) {
+ try {
+ return this.speakerTrackers.get(source)?.identify(samples) ?? null;
+ } catch (e) {
+ getLog().error(`Transcription: identifying the speaker on ${source} failed: ${e}`);
+ return null;
+ }
+ }
+
/** Reports the sources with speech not yet written, when they differ from the last report. */
private updatePending() {
const pending = [ ...this.pipelines ]
diff --git a/apps/desktop/src/services/transcription/speakers.spec.ts b/apps/desktop/src/services/transcription/speakers.spec.ts
new file mode 100644
index 0000000000..0ae07f6785
--- /dev/null
+++ b/apps/desktop/src/services/transcription/speakers.spec.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+
+import { SpeakerClusterer } from "./speakers";
+
+const options = { threshold: 0.5, minDurationSeconds: 1.5 };
+
+describe("SpeakerClusterer", () => {
+ it("joins similar embeddings and starts a speaker for a distant one", () => {
+ const clusterer = new SpeakerClusterer(options);
+
+ expect(clusterer.assign(new Float32Array([ 1, 0, 0 ]), 3)).toBe(1);
+ // Scale does not matter, only direction.
+ expect(clusterer.assign(new Float32Array([ 9, 1, 0 ]), 3)).toBe(1);
+ expect(clusterer.assign(new Float32Array([ 0, 1, 0 ]), 3)).toBe(2);
+ expect(clusterer.assign(new Float32Array([ 0.1, 1, 0 ]), 3)).toBe(2);
+ expect(clusterer.assign(new Float32Array([ 0, 0, 1 ]), 3)).toBe(3);
+ });
+
+ it("lets a short utterance join a speaker but never start one or move a centroid", () => {
+ const clusterer = new SpeakerClusterer(options);
+ expect(clusterer.assign(new Float32Array([ 0, 1 ]), 1)).toBeNull();
+ expect(clusterer.assign(new Float32Array([ 1, 0 ]), 3)).toBe(1);
+
+ // Cosine 0.71 joins speaker 1; a centroid moved by it would let [0, 1] join too.
+ expect(clusterer.assign(new Float32Array([ 1, 1 ]), 1)).toBe(1);
+ expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 1)).toBeNull();
+ expect(clusterer.assign(new Float32Array([ 0.2, 1 ]), 2)).toBe(2);
+ });
+
+ it("moves a centroid with every long utterance it takes", () => {
+ const clusterer = new SpeakerClusterer(options);
+ clusterer.assign(new Float32Array([ 1, 0 ]), 3);
+ // Cosine 0.6 with [1, 0] joins, and pulls the centroid toward [0.6, 0.8].
+ expect(clusterer.assign(new Float32Array([ 0.6, 0.8 ]), 3)).toBe(1);
+ // Cosine 0.28 with [1, 0] alone; 0.68 with the moved centroid.
+ expect(clusterer.assign(new Float32Array([ 0.28, 0.96 ]), 3)).toBe(1);
+ });
+});
diff --git a/apps/desktop/src/services/transcription/speakers.ts b/apps/desktop/src/services/transcription/speakers.ts
new file mode 100644
index 0000000000..d0ead044bc
--- /dev/null
+++ b/apps/desktop/src/services/transcription/speakers.ts
@@ -0,0 +1,67 @@
+export interface SpeakerClustererOptions {
+ /** The cosine similarity to a speaker's centroid from which an utterance joins that speaker. */
+ threshold: number;
+ /** Utterances shorter than this, in seconds, never start a speaker nor move a centroid. */
+ minDurationSeconds: number;
+}
+
+/**
+ * Assigns embeddings to speakers as they arrive: an embedding joins the speaker whose centroid it
+ * is most similar to, when that similarity reaches the threshold, and starts a new speaker
+ * otherwise.
+ */
+export class SpeakerClusterer {
+ /** The sum of each speaker's normalized embeddings; its direction is the centroid. */
+ private readonly centroids: Float32Array[] = [];
+
+ constructor(private readonly options: SpeakerClustererOptions) {}
+
+ assign(embedding: Float32Array, durationSeconds: number): number | null {
+ const normalized = normalize(embedding);
+ let best = -1;
+ let bestSimilarity = -Infinity;
+ for (const [ index, centroid ] of this.centroids.entries()) {
+ const similarity = dot(normalize(centroid), normalized);
+ if (similarity > bestSimilarity) {
+ best = index;
+ bestSimilarity = similarity;
+ }
+ }
+
+ const isLongEnough = durationSeconds >= this.options.minDurationSeconds;
+ if (best >= 0 && bestSimilarity >= this.options.threshold) {
+ if (isLongEnough) {
+ addInto(this.centroids[best], normalized);
+ }
+ return best + 1;
+ }
+ if (!isLongEnough) {
+ return null;
+ }
+ this.centroids.push(normalized);
+ return this.centroids.length;
+ }
+}
+
+function normalize(vector: Float32Array) {
+ let sumOfSquares = 0;
+ for (const value of vector) {
+ sumOfSquares += value * value;
+ }
+ const length = Math.sqrt(sumOfSquares) || 1;
+ return vector.map((value) => value / length);
+}
+
+function dot(a: Float32Array, b: Float32Array) {
+ let sum = 0;
+ for (const [ index, value ] of a.entries()) {
+ sum += value * b[index];
+ }
+ return sum;
+}
+
+function addInto(target: Float32Array, vector: Float32Array) {
+ for (const [ index, value ] of vector.entries()) {
+ target[index] += value;
+ }
+}
diff --git a/apps/desktop/src/sherpa-onnx-node.d.ts b/apps/desktop/src/sherpa-onnx-node.d.ts
index 5f9ce1def2..4cf8ce2841 100644
--- a/apps/desktop/src/sherpa-onnx-node.d.ts
+++ b/apps/desktop/src/sherpa-onnx-node.d.ts
@@ -9,6 +9,18 @@ declare module "sherpa-onnx-node" {
decodeAsync(stream: OfflineStream): Promise<{ text: string }>;
}
+ export interface SpeakerEmbeddingStream {
+ acceptWaveform(wave: { samples: Float32Array; sampleRate: number }): void;
+ inputFinished(): void;
+ }
+
+ export class SpeakerEmbeddingExtractor {
+ constructor(config: object);
+ createStream(): SpeakerEmbeddingStream;
+ isReady(stream: SpeakerEmbeddingStream): boolean;
+ compute(stream: SpeakerEmbeddingStream, enableExternalBuffer: boolean): Float32Array;
+ }
+
export class Vad {
constructor(config: object, bufferSizeInSeconds: number);
acceptWaveform(samples: Float32Array): void;
diff --git a/packages/commons/src/lib/transcript.ts b/packages/commons/src/lib/transcript.ts
index a49f90becb..a2100912d0 100644
--- a/packages/commons/src/lib/transcript.ts
+++ b/packages/commons/src/lib/transcript.ts
@@ -11,6 +11,11 @@ export interface TranscriptUtterance {
offsetMs: number;
/** Length of the utterance in milliseconds. Absent from transcripts recorded before it was kept. */
durationMs?: number;
+ /**
+ * Who spoke, numbered from 1 within the source in the order the speakers were first heard.
+ * Absent when the speaker could not be told, and on the microphone.
+ */
+ speaker?: number;
text: string;
}