Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

Meeting notes: enable live speaker identification on Linux (#2453)

Filip Stachnik63c3362ea872
diff --git a/src/helpers/liveSpeakerIdPolicy.js b/src/helpers/liveSpeakerIdPolicy.js
index 1efbd54c0..cd825bf47 100644
--- a/src/helpers/liveSpeakerIdPolicy.js
+++ b/src/helpers/liveSpeakerIdPolicy.js
@@ -4,9 +4,9 @@
// "native" (macOS Core Audio tap) and Windows "loopback" (WASAPI helper or
// renderer display-media fallback) all deliver 24 kHz mono s16le PCM through
// the same sendMeetingAudio("system") path, so the identifier works unchanged.
-// Linux reports "loopback" too (PipeWire portal) but that capture path has not
-// been verified against the identifier yet, so it stays disabled.
+// Linux reports "loopback" too (PipeWire helper) and feeds the same stream;
+// the identifier regroups buffers of any size for its resampler.
export function supportsLiveSpeakerIdentification(systemAudioMode, platform = process.platform) {
if (systemAudioMode === "native") return true;
- return systemAudioMode === "loopback" && platform === "win32";
+ return systemAudioMode === "loopback" && (platform === "win32" || platform === "linux");
}
diff --git a/src/helpers/liveSpeakerIdentifier.js b/src/helpers/liveSpeakerIdentifier.js
index 991fb9483..90da49fdc 100644
--- a/src/helpers/liveSpeakerIdentifier.js
+++ b/src/helpers/liveSpeakerIdentifier.js
@@ -48,6 +48,12 @@ const SILENCE_WINDOWS_TO_END = 24;
const MATCH_THRESHOLD = 0.65;
const MATCH_MARGIN = 0.03;
const LIVE_WINDOW_PADDING_SECONDS = 0.75;
+// The 24 kHz -> 16 kHz resampler restarts on every buffer and only divides
+// evenly on whole groups of 3 input samples (6 bytes). Capture buffers come in
+// arbitrary sizes (PipeWire quantum, coalesced pipe reads, renderer loopback
+// frames, recovery silence), so a partial group is carried into the next
+// buffer; dropping it pulls the identifier clock behind real time.
+const RESAMPLE_GROUP_BYTES = 6;
const DEFAULT_VAD_STATE_SHAPE = [2, 1, 64];
// The VAD graph is tiny (~64 ops per 512-sample window at ~31 Hz); ORT's
// default thread pool spins one worker per core for no throughput gain.
@@ -145,6 +151,7 @@ class LiveSpeakerIdentifier {
this.queue = Promise.resolve();
this.onSpeakerIdentified = null;
this.getSpeakerProfiles = null;
+ this.pcmRemainder = Buffer.alloc(0);
this.audioRemainder = new Float32Array(0);
this.vadStateInputs = [];
this.vadStateOutputs = [];
@@ -420,6 +427,7 @@ class LiveSpeakerIdentifier {
_resetMeetingState() {
this.queue = Promise.resolve();
+ this.pcmRemainder = Buffer.alloc(0);
this.audioRemainder = new Float32Array(0);
this.speechChunks = [];
this.speechActive = false;
@@ -458,9 +466,16 @@ class LiveSpeakerIdentifier {
await this._ensureLoaded();
if (!this.session) return;
- const downsampled = downsample24kTo16k(
- Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer)
- );
+ // Carry the bytes past the last whole resample group into the next buffer.
+ // The concat copies into a new buffer, so the Int16 view never inherits the
+ // caller's byteOffset.
+ const incoming = Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer);
+ const pending = Buffer.concat([this.pcmRemainder, incoming]);
+ const usableBytes = pending.length - (pending.length % RESAMPLE_GROUP_BYTES);
+ this.pcmRemainder = Buffer.from(pending.subarray(usableBytes));
+ if (!usableBytes) return;
+
+ const downsampled = downsample24kTo16k(pending.subarray(0, usableBytes));
if (!downsampled.length) return;
this.audioRemainder = appendFloat32(this.audioRemainder, pcm16ToFloat32(downsampled));
diff --git a/test/helpers/liveSpeakerIdPolicy.test.js b/test/helpers/liveSpeakerIdPolicy.test.js
index 58903b193..45a69b548 100644
--- a/test/helpers/liveSpeakerIdPolicy.test.js
+++ b/test/helpers/liveSpeakerIdPolicy.test.js
@@ -20,10 +20,11 @@ test("Windows loopback supports live speaker identification", async () => {
assert.equal(supportsLiveSpeakerIdentification("loopback", "win32"), true);
});
-test("Linux loopback (PipeWire portal) stays disabled until verified", async () => {
+test("Linux loopback (PipeWire portal) supports live speaker identification", async () => {
const { supportsLiveSpeakerIdentification } = await load();
- assert.equal(supportsLiveSpeakerIdentification("loopback", "linux"), false);
+ // The helper feeds the same 24 kHz mono s16le sendMeetingAudio path.
+ assert.equal(supportsLiveSpeakerIdentification("loopback", "linux"), true);
});
test("unsupported mode never identifies speakers", async () => {
@@ -63,7 +64,7 @@ test("platform defaults to the current process platform", async () => {
assert.equal(supportsLiveSpeakerIdentification("native"), true);
});
onPlatform("linux", () => {
- assert.equal(supportsLiveSpeakerIdentification("loopback"), false);
+ assert.equal(supportsLiveSpeakerIdentification("loopback"), true);
assert.equal(supportsLiveSpeakerIdentification("native"), true);
});
onPlatform("darwin", () => {
diff --git a/test/helpers/liveSpeakerIdentifier.test.js b/test/helpers/liveSpeakerIdentifier.test.js
index 2f7ca11fd..61dfdfad6 100644
--- a/test/helpers/liveSpeakerIdentifier.test.js
+++ b/test/helpers/liveSpeakerIdentifier.test.js
@@ -1,6 +1,7 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const Module = require("node:module");
+const { downsample24kTo16k, pcm16ToFloat32 } = require("../../src/utils/audioUtils");
const identifierModulePath = require.resolve("../../src/helpers/liveSpeakerIdentifier");
const originalLoad = Module._load;
@@ -421,3 +422,73 @@ test("distinct voices are folded into one cluster when the session cap is 1", ()
"at the cap, new voices are force-merged into the nearest cluster by design"
);
});
+
+// Every 16 kHz sample the identifier produced, in order: the VAD windows it
+// processed plus what is still buffered.
+async function collect16kStream(identifier, chunks) {
+ const windows = [];
+ identifier.session = {}; // loaded: skip _ensureLoaded
+ identifier._processWindow = async (window) => windows.push(Float32Array.from(window));
+ for (const chunk of chunks) await identifier._processAudio(chunk);
+
+ const stream = new Float32Array(identifier.sampleCursor + identifier.audioRemainder.length);
+ windows.forEach((window, index) => stream.set(window, index * window.length));
+ stream.set(identifier.audioRemainder, identifier.sampleCursor);
+ return stream;
+}
+
+// Capture buffers come in arbitrary sizes: the Linux helper's 485 then 512
+// samples, coalesced pipe reads, samples split across reads, and one buffer at
+// an odd byteOffset.
+function irregularChunks(source) {
+ const sizes = [970, 1024, 1023, 1025, 1, 5, 4800, 1024];
+ const chunks = [];
+ for (let offset = 0, i = 0; offset < source.length; i++) {
+ const size = sizes[i % sizes.length];
+ chunks.push(source.subarray(offset, offset + size));
+ offset += size;
+ }
+ const odd = Buffer.alloc(chunks[1].length + 1).subarray(1);
+ chunks[1].copy(odd);
+ chunks[1] = odd;
+ return chunks;
+}
+
+// assert.deepEqual on 70k-sample arrays builds an enormous diff on failure.
+function firstMismatch(a, b) {
+ const length = Math.max(a.length, b.length);
+ for (let i = 0; i < length; i++) if (a[i] !== b[i]) return i;
+ return -1;
+}
+
+function makePcm(samples) {
+ const pcm = Buffer.alloc(samples * 2);
+ for (let i = 0; i < samples; i++) pcm.writeInt16LE(((i * 7919) % 60000) - 30000, i * 2);
+ return pcm;
+}
+
+test("audio in irregular buffers resamples to the same 16 kHz stream as one buffer", async () => {
+ const { LiveSpeakerIdentifier } = loadIdentifier();
+ // A whole number of resample groups, so nothing is left carried at the end.
+ const source = makePcm(485 + 512 * 200);
+ const expected = pcm16ToFloat32(downsample24kTo16k(source));
+
+ const stream = await collect16kStream(new LiveSpeakerIdentifier(), irregularChunks(source));
+
+ assert.equal(stream.length, expected.length, "no 16 kHz sample is lost or gained");
+ assert.equal(firstMismatch(stream, expected), -1, "same samples, in order");
+});
+
+test("a new meeting never inherits the previous meeting's partial resample group", async () => {
+ const { LiveSpeakerIdentifier } = loadIdentifier();
+ const reused = new LiveSpeakerIdentifier();
+ reused.session = {};
+ await reused._processAudio(Buffer.alloc(4, 0x7f));
+ reused._resetMeetingState();
+
+ const chunks = irregularChunks(makePcm(512 * 30));
+ const afterReset = await collect16kStream(reused, chunks);
+ const fresh = await collect16kStream(new LiveSpeakerIdentifier(), chunks);
+
+ assert.equal(firstMismatch(afterReset, fresh), -1);
+});