Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
Meeting notes: enable live speaker identification on Linux (#2453)
Filip Stachnik63c3362ea872
diff --git a/src/helpers/liveSpeakerIdPolicy.js b/src/helpers/liveSpeakerIdPolicy.jsindex 1efbd54c0..cd825bf47 100644--- a/src/helpers/liveSpeakerIdPolicy.js+++ b/src/helpers/liveSpeakerIdPolicy.js@@ -4,9 +4,9 @@// "native" (macOS Core Audio tap) and Windows "loopback" (WASAPI helper or// renderer display-media fallback) all deliver 24 kHz mono s16le PCM through// the same sendMeetingAudio("system") path, so the identifier works unchanged.-// Linux reports "loopback" too (PipeWire portal) but that capture path has not-// been verified against the identifier yet, so it stays disabled.+// Linux reports "loopback" too (PipeWire helper) and feeds the same stream;+// the identifier regroups buffers of any size for its resampler.export function supportsLiveSpeakerIdentification(systemAudioMode, platform = process.platform) {if (systemAudioMode === "native") return true;- return systemAudioMode === "loopback" && platform === "win32";+ return systemAudioMode === "loopback" && (platform === "win32" || platform === "linux");}diff --git a/src/helpers/liveSpeakerIdentifier.js b/src/helpers/liveSpeakerIdentifier.jsindex 991fb9483..90da49fdc 100644--- a/src/helpers/liveSpeakerIdentifier.js+++ b/src/helpers/liveSpeakerIdentifier.js@@ -48,6 +48,12 @@ const SILENCE_WINDOWS_TO_END = 24;const MATCH_THRESHOLD = 0.65;const MATCH_MARGIN = 0.03;const LIVE_WINDOW_PADDING_SECONDS = 0.75;+// The 24 kHz -> 16 kHz resampler restarts on every buffer and only divides+// evenly on whole groups of 3 input samples (6 bytes). Capture buffers come in+// arbitrary sizes (PipeWire quantum, coalesced pipe reads, renderer loopback+// frames, recovery silence), so a partial group is carried into the next+// buffer; dropping it pulls the identifier clock behind real time.+const RESAMPLE_GROUP_BYTES = 6;const DEFAULT_VAD_STATE_SHAPE = [2, 1, 64];// The VAD graph is tiny (~64 ops per 512-sample window at ~31 Hz); ORT's// default thread pool spins one worker per core for no throughput gain.@@ -145,6 +151,7 @@ class LiveSpeakerIdentifier {this.queue = Promise.resolve();this.onSpeakerIdentified = null;this.getSpeakerProfiles = null;+ this.pcmRemainder = Buffer.alloc(0);this.audioRemainder = new Float32Array(0);this.vadStateInputs = [];this.vadStateOutputs = [];@@ -420,6 +427,7 @@ class LiveSpeakerIdentifier {_resetMeetingState() {this.queue = Promise.resolve();+ this.pcmRemainder = Buffer.alloc(0);this.audioRemainder = new Float32Array(0);this.speechChunks = [];this.speechActive = false;@@ -458,9 +466,16 @@ class LiveSpeakerIdentifier {await this._ensureLoaded();if (!this.session) return;- const downsampled = downsample24kTo16k(- Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer)- );+ // Carry the bytes past the last whole resample group into the next buffer.+ // The concat copies into a new buffer, so the Int16 view never inherits the+ // caller's byteOffset.+ const incoming = Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer);+ const pending = Buffer.concat([this.pcmRemainder, incoming]);+ const usableBytes = pending.length - (pending.length % RESAMPLE_GROUP_BYTES);+ this.pcmRemainder = Buffer.from(pending.subarray(usableBytes));+ if (!usableBytes) return;++ const downsampled = downsample24kTo16k(pending.subarray(0, usableBytes));if (!downsampled.length) return;this.audioRemainder = appendFloat32(this.audioRemainder, pcm16ToFloat32(downsampled));diff --git a/test/helpers/liveSpeakerIdPolicy.test.js b/test/helpers/liveSpeakerIdPolicy.test.jsindex 58903b193..45a69b548 100644--- a/test/helpers/liveSpeakerIdPolicy.test.js+++ b/test/helpers/liveSpeakerIdPolicy.test.js@@ -20,10 +20,11 @@ test("Windows loopback supports live speaker identification", async () => {assert.equal(supportsLiveSpeakerIdentification("loopback", "win32"), true);});-test("Linux loopback (PipeWire portal) stays disabled until verified", async () => {+test("Linux loopback (PipeWire portal) supports live speaker identification", async () => {const { supportsLiveSpeakerIdentification } = await load();- assert.equal(supportsLiveSpeakerIdentification("loopback", "linux"), false);+ // The helper feeds the same 24 kHz mono s16le sendMeetingAudio path.+ assert.equal(supportsLiveSpeakerIdentification("loopback", "linux"), true);});test("unsupported mode never identifies speakers", async () => {@@ -63,7 +64,7 @@ test("platform defaults to the current process platform", async () => {assert.equal(supportsLiveSpeakerIdentification("native"), true);});onPlatform("linux", () => {- assert.equal(supportsLiveSpeakerIdentification("loopback"), false);+ assert.equal(supportsLiveSpeakerIdentification("loopback"), true);assert.equal(supportsLiveSpeakerIdentification("native"), true);});onPlatform("darwin", () => {diff --git a/test/helpers/liveSpeakerIdentifier.test.js b/test/helpers/liveSpeakerIdentifier.test.jsindex 2f7ca11fd..61dfdfad6 100644--- a/test/helpers/liveSpeakerIdentifier.test.js+++ b/test/helpers/liveSpeakerIdentifier.test.js@@ -1,6 +1,7 @@const test = require("node:test");const assert = require("node:assert/strict");const Module = require("node:module");+const { downsample24kTo16k, pcm16ToFloat32 } = require("../../src/utils/audioUtils");const identifierModulePath = require.resolve("../../src/helpers/liveSpeakerIdentifier");const originalLoad = Module._load;@@ -421,3 +422,73 @@ test("distinct voices are folded into one cluster when the session cap is 1", ()"at the cap, new voices are force-merged into the nearest cluster by design");});++// Every 16 kHz sample the identifier produced, in order: the VAD windows it+// processed plus what is still buffered.+async function collect16kStream(identifier, chunks) {+ const windows = [];+ identifier.session = {}; // loaded: skip _ensureLoaded+ identifier._processWindow = async (window) => windows.push(Float32Array.from(window));+ for (const chunk of chunks) await identifier._processAudio(chunk);++ const stream = new Float32Array(identifier.sampleCursor + identifier.audioRemainder.length);+ windows.forEach((window, index) => stream.set(window, index * window.length));+ stream.set(identifier.audioRemainder, identifier.sampleCursor);+ return stream;+}++// Capture buffers come in arbitrary sizes: the Linux helper's 485 then 512+// samples, coalesced pipe reads, samples split across reads, and one buffer at+// an odd byteOffset.+function irregularChunks(source) {+ const sizes = [970, 1024, 1023, 1025, 1, 5, 4800, 1024];+ const chunks = [];+ for (let offset = 0, i = 0; offset < source.length; i++) {+ const size = sizes[i % sizes.length];+ chunks.push(source.subarray(offset, offset + size));+ offset += size;+ }+ const odd = Buffer.alloc(chunks[1].length + 1).subarray(1);+ chunks[1].copy(odd);+ chunks[1] = odd;+ return chunks;+}++// assert.deepEqual on 70k-sample arrays builds an enormous diff on failure.+function firstMismatch(a, b) {+ const length = Math.max(a.length, b.length);+ for (let i = 0; i < length; i++) if (a[i] !== b[i]) return i;+ return -1;+}++function makePcm(samples) {+ const pcm = Buffer.alloc(samples * 2);+ for (let i = 0; i < samples; i++) pcm.writeInt16LE(((i * 7919) % 60000) - 30000, i * 2);+ return pcm;+}++test("audio in irregular buffers resamples to the same 16 kHz stream as one buffer", async () => {+ const { LiveSpeakerIdentifier } = loadIdentifier();+ // A whole number of resample groups, so nothing is left carried at the end.+ const source = makePcm(485 + 512 * 200);+ const expected = pcm16ToFloat32(downsample24kTo16k(source));++ const stream = await collect16kStream(new LiveSpeakerIdentifier(), irregularChunks(source));++ assert.equal(stream.length, expected.length, "no 16 kHz sample is lost or gained");+ assert.equal(firstMismatch(stream, expected), -1, "same samples, in order");+});++test("a new meeting never inherits the previous meeting's partial resample group", async () => {+ const { LiveSpeakerIdentifier } = loadIdentifier();+ const reused = new LiveSpeakerIdentifier();+ reused.session = {};+ await reused._processAudio(Buffer.alloc(4, 0x7f));+ reused._resetMeetingState();++ const chunks = irregularChunks(makePcm(512 * 30));+ const afterReset = await collect16kStream(reused, chunks);+ const fresh = await collect16kStream(new LiveSpeakerIdentifier(), chunks);++ assert.equal(firstMismatch(afterReset, fresh), -1);+});