Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat(parakeet): add NVIDIA Nemotron streaming ASR models (EN + multilingual 3.5) (#1131)

Gabriel Steine4e5d19ddb35
diff --git a/CLAUDE.md b/CLAUDE.md
index 9865eb85f..550876cfe 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -179,6 +179,12 @@ OpenWhispr is an Electron-based desktop dictation application that uses whisper.
- **Available Models**:
- `parakeet-tdt-0.6b-v3`: Multilingual (25 languages), ~680MB
- `parakeet-unified-en-0.6b`: English-only, ~631MB, state-of-the-art EN accuracy (5.91% avg WER on Open ASR Leaderboard)
+ - `nemotron-speech-streaming-en-0.6b`: English-only, ~632MB, cache-aware streaming FastConformer (`"runtime": "online"` in the registry)
+ - `nemotron-3.5-asr-streaming-0.6b`: Multilingual (15 transcription-ready languages, auto detection), ~650MB, cache-aware streaming FastConformer (`"runtime": "online"`)
+
+- **Runtimes**: Models are `offline` (default) or `online` per their registry `runtime` field. Offline models use the bundled `sherpa-onnx-ws-{platform}-{arch}` (offline websocket server); online models use `sherpa-onnx-online-ws-{platform}-{arch}` (online websocket server). Both are downloaded by `scripts/download-sherpa-onnx.js`. The final transcription path still records-then-transcribes; audio is chunked over the websocket and partial/final JSON results are merged by `parakeetWsResult.js`.
+
+- **Live Transcription Preview**: When the preview toggle is on and an online-runtime model is selected, the preview uses a persistent websocket stream (`createOnlineStream` in `parakeetWsServer.js`): worklet PCM is fed as it is captured (`sendPcm16` converts to the float32 wire format inside the ws layer), and partial results update the preview window live (replacing text via `showTranscriptionPreview`). Offline models keep the 1.5s buffered-chunk path (appending via `appendTranscriptionPreview`). If the stream can't start, the preview falls back to the chunked path. Tests: `test/helpers/parakeetOnlineStream.test.js` (mock websocket server).
- **Download URLs**: Models from sherpa-onnx ASR models release on GitHub
diff --git a/package.json b/package.json
index f8a2f3862..834a84521 100644
--- a/package.json
+++ b/package.json
@@ -20,11 +20,11 @@
"compile:mic-listener": "node scripts/build-macos-mic-listener.js",
"compile:audio-tap": "node scripts/build-macos-audio-tap.js",
"compile:native": "npm run compile:globe && npm run compile:fast-paste && npm run compile:winkeys && npm run compile:linuxkeys && npm run compile:winpaste && npm run compile:linux-paste && npm run compile:linux-system-audio && npm run compile:text-monitor && npm run compile:media-remote && npm run compile:mic-listener && npm run compile:audio-tap",
- "prestart": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",
+ "prestart": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",
"start": "electron .",
- "predev": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",
+ "predev": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",
"dev": "concurrently -k -r \"npm:dev:renderer\" \"npm:dev:main\"",
- "predev:main": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",
+ "predev:main": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",
"dev:main": "cross-env NODE_ENV=development node scripts/run-electron.js --dev",
"dev:renderer": "cd src && vite",
"prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:meeting-aec-helper && npm run download:whisper-vad-model && npm run download:embedding-model -- --for-build && npm run download:diarization-models -- --output-dir resources/bin/diarization-models",
diff --git a/scripts/download-sherpa-onnx.js b/scripts/download-sherpa-onnx.js
index 5d072bd5c..8778cef73 100644
--- a/scripts/download-sherpa-onnx.js
+++ b/scripts/download-sherpa-onnx.js
@@ -2,9 +2,15 @@
const fs = require("fs");
const path = require("path");
const { execFileSync } = require("child_process");
-const { downloadFile, findBinaryInDir, parseArgs, setExecutable } = require("./lib/download-utils");
-
-const SHERPA_ONNX_VERSION = "1.12.23";
+const {
+ cleanupFiles,
+ downloadFile,
+ findBinaryInDir,
+ parseArgs,
+ setExecutable,
+} = require("./lib/download-utils");
+
+const SHERPA_ONNX_VERSION = "1.13.4";
const GITHUB_RELEASE_URL = `https://github.com/k2-fsa/sherpa-onnx/releases/download/v${SHERPA_ONNX_VERSION}`;
// Binary configurations for each platform
@@ -14,6 +20,8 @@ const BINARIES = {
archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-osx-universal2-shared.tar.bz2`,
binaryPath: "sherpa-onnx-offline-websocket-server",
outputName: "sherpa-onnx-ws-darwin-arm64",
+ onlineBinaryPath: "sherpa-onnx-online-websocket-server",
+ onlineOutputName: "sherpa-onnx-online-ws-darwin-arm64",
diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization",
diarizeOutputName: "sherpa-onnx-diarize-darwin-arm64",
libPattern: "*.dylib",
@@ -22,14 +30,19 @@ const BINARIES = {
archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-osx-universal2-shared.tar.bz2`,
binaryPath: "sherpa-onnx-offline-websocket-server",
outputName: "sherpa-onnx-ws-darwin-x64",
+ onlineBinaryPath: "sherpa-onnx-online-websocket-server",
+ onlineOutputName: "sherpa-onnx-online-ws-darwin-x64",
diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization",
diarizeOutputName: "sherpa-onnx-diarize-darwin-x64",
libPattern: "*.dylib",
},
"win32-x64": {
- archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-win-x64-shared.tar.bz2`,
+ // Since 1.13.4 the Windows assets carry an MSVC runtime/build-type suffix
+ archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-win-x64-shared-MD-Release.tar.bz2`,
binaryPath: "sherpa-onnx-offline-websocket-server.exe",
outputName: "sherpa-onnx-ws-win32-x64.exe",
+ onlineBinaryPath: "sherpa-onnx-online-websocket-server.exe",
+ onlineOutputName: "sherpa-onnx-online-ws-win32-x64.exe",
diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization.exe",
diarizeOutputName: "sherpa-onnx-diarize-win32-x64.exe",
libPattern: "*.dll",
@@ -38,6 +51,8 @@ const BINARIES = {
archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-linux-x64-shared.tar.bz2`,
binaryPath: "sherpa-onnx-offline-websocket-server",
outputName: "sherpa-onnx-ws-linux-x64",
+ onlineBinaryPath: "sherpa-onnx-online-websocket-server",
+ onlineOutputName: "sherpa-onnx-online-ws-linux-x64",
diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization",
diarizeOutputName: "sherpa-onnx-diarize-linux-x64",
libPattern: "*.so*",
@@ -46,6 +61,14 @@ const BINARIES = {
const BIN_DIR = path.join(__dirname, "..", "resources", "bin");
+const VERSIONED_LIB_PATTERN = /^(lib.+?)\.(\d+\.\d+\.\d+)\.(dylib|so|dll)$/;
+
+// Upstream 1.13.4 ships an invalid arm64 signature on libonnxruntime; dyld SIGKILLs unsigned loads.
+function adhocSign(filePath, platformArch) {
+ if (process.platform !== "darwin" || !platformArch.startsWith("darwin")) return;
+ execFileSync("codesign", ["--force", "--sign", "-", filePath], { stdio: "ignore" });
+}
+
function getDownloadUrl(archiveName) {
return `${GITHUB_RELEASE_URL}/${archiveName}`;
}
@@ -95,6 +118,37 @@ function matchesPattern(filename, pattern) {
return false;
}
+function copyBinary(extractDir, binaryName, outputPath, platformArch) {
+ const foundPath = findBinaryInDir(extractDir, binaryName);
+
+ if (!foundPath || !fs.existsSync(foundPath)) {
+ console.error(` ${platformArch}: Binary '${binaryName}' not found in archive`);
+ return false;
+ }
+
+ fs.rmSync(outputPath, { force: true });
+ fs.copyFileSync(foundPath, outputPath);
+ setExecutable(outputPath);
+ adhocSign(outputPath, platformArch);
+ console.log(` ${platformArch}: Extracted to ${path.basename(outputPath)}`);
+ return true;
+}
+
+function isCompleteInstall(markerPath, binaryPaths) {
+ if (binaryPaths.some((binaryPath) => !fs.existsSync(binaryPath))) return false;
+
+ try {
+ const marker = JSON.parse(fs.readFileSync(markerPath, "utf8"));
+ return (
+ marker.version === SHERPA_ONNX_VERSION &&
+ Array.isArray(marker.libraries) &&
+ marker.libraries.every((lib) => fs.existsSync(path.join(BIN_DIR, lib)))
+ );
+ } catch {
+ return false;
+ }
+}
+
async function downloadBinary(platformArch, config, isForce = false) {
if (!config) {
console.log(` ${platformArch}: Not supported`);
@@ -102,54 +156,38 @@ async function downloadBinary(platformArch, config, isForce = false) {
}
const outputPath = path.join(BIN_DIR, config.outputName);
+ const onlineOutputPath = path.join(BIN_DIR, config.onlineOutputName);
const diarizeOutputPath = path.join(BIN_DIR, config.diarizeOutputName);
+ const installMarkerPath = path.join(BIN_DIR, `.sherpa-onnx-${platformArch}.json`);
- if (fs.existsSync(outputPath) && fs.existsSync(diarizeOutputPath) && !isForce) {
+ if (!isForce && isCompleteInstall(installMarkerPath, [outputPath, onlineOutputPath, diarizeOutputPath])) {
console.log(` ${platformArch}: Already exists (use --force to re-download)`);
return true;
}
+ if (isForce && fs.existsSync(installMarkerPath)) fs.unlinkSync(installMarkerPath);
const url = getDownloadUrl(config.archiveName);
console.log(` ${platformArch}: Downloading from ${url}`);
const archivePath = path.join(BIN_DIR, config.archiveName);
+ const extractDir = path.join(BIN_DIR, `temp-sherpa-${platformArch}`);
try {
await downloadFile(url, archivePath);
- const extractDir = path.join(BIN_DIR, `temp-sherpa-${platformArch}`);
fs.mkdirSync(extractDir, { recursive: true });
extractTarBz2(archivePath, extractDir);
- // Find the WebSocket server binary (may be in a subdirectory)
- const binaryName = path.basename(config.binaryPath);
- let binaryPath = findBinaryInDir(extractDir, binaryName);
-
- if (binaryPath && fs.existsSync(binaryPath)) {
- fs.copyFileSync(binaryPath, outputPath);
- setExecutable(outputPath);
- console.log(` ${platformArch}: Extracted to ${config.outputName}`);
- } else {
- console.error(` ${platformArch}: Binary '${binaryName}' not found in archive`);
- return false;
- }
-
- // Find the diarization binary from the same archive
- const diarizeBinaryName = path.basename(config.diarizeBinaryPath);
- let diarizeBinaryPath = findBinaryInDir(extractDir, diarizeBinaryName);
-
- if (diarizeBinaryPath && fs.existsSync(diarizeBinaryPath)) {
- fs.copyFileSync(diarizeBinaryPath, diarizeOutputPath);
- setExecutable(diarizeOutputPath);
- console.log(` ${platformArch}: Extracted to ${config.diarizeOutputName}`);
- } else {
- console.error(
- ` ${platformArch}: Diarization binary '${diarizeBinaryName}' not found in archive`
- );
- return false;
+ for (const [binaryName, destPath] of [
+ [config.binaryPath, outputPath],
+ [config.onlineBinaryPath, onlineOutputPath],
+ [config.diarizeBinaryPath, diarizeOutputPath],
+ ]) {
+ if (!copyBinary(extractDir, binaryName, destPath, platformArch)) return false;
}
// Copy shared libraries
+ const copiedLibraries = [];
if (config.libPattern) {
const libraries = findLibrariesInDir(extractDir, config.libPattern);
@@ -161,15 +199,17 @@ async function downloadBinary(platformArch, config, isForce = false) {
const libName = path.basename(libPath);
const destPath = path.join(BIN_DIR, libName);
- // Detect versioned dylib pattern: libFoo.X.Y.Z.dylib
- const versionMatch = libName.match(/^(lib.+?)\.(\d+\.\d+\.\d+)\.(dylib|so|dll)$/);
+ const versionMatch = libName.match(VERSIONED_LIB_PATTERN);
if (versionMatch) {
- const baseName = `${versionMatch[1]}.${versionMatch[3]}`; // e.g. libonnxruntime.dylib
- versionedLibs.set(baseName, libName);
+ versionedLibs.set(`${versionMatch[1]}.${versionMatch[3]}`, libName);
}
+ // rm first: copying onto an existing symlink would write through it
+ fs.rmSync(destPath, { force: true });
fs.copyFileSync(libPath, destPath);
setExecutable(destPath);
+ adhocSign(destPath, platformArch);
+ copiedLibraries.push(libName);
console.log(` ${platformArch}: Copied library ${libName}`);
}
@@ -177,28 +217,32 @@ async function downloadBinary(platformArch, config, isForce = false) {
if (process.platform !== "win32") {
for (const [baseName, versionedName] of versionedLibs) {
const basePath = path.join(BIN_DIR, baseName);
- const versionedPath = path.join(BIN_DIR, versionedName);
- if (
- fs.existsSync(basePath) &&
- fs.existsSync(versionedPath) &&
- !fs.lstatSync(basePath).isSymbolicLink()
- ) {
- fs.unlinkSync(basePath);
- fs.symlinkSync(versionedName, basePath);
- console.log(` ${platformArch}: Symlinked ${baseName} -> ${versionedName}`);
+ fs.rmSync(basePath, { force: true });
+ fs.symlinkSync(versionedName, basePath);
+ console.log(` ${platformArch}: Symlinked ${baseName} -> ${versionedName}`);
+
+ for (const file of fs.readdirSync(BIN_DIR)) {
+ const match = file.match(VERSIONED_LIB_PATTERN);
+ if (match && `${match[1]}.${match[3]}` === baseName && file !== versionedName) {
+ fs.unlinkSync(path.join(BIN_DIR, file));
+ console.log(` ${platformArch}: Removed stale ${file}`);
+ }
}
}
}
}
- // Cleanup
- fs.rmSync(extractDir, { recursive: true, force: true });
- if (fs.existsSync(archivePath)) fs.unlinkSync(archivePath);
+ fs.writeFileSync(
+ installMarkerPath,
+ JSON.stringify({ version: SHERPA_ONNX_VERSION, libraries: copiedLibraries })
+ );
return true;
} catch (error) {
console.error(` ${platformArch}: Failed - ${error.message}`);
- if (fs.existsSync(archivePath)) fs.unlinkSync(archivePath);
return false;
+ } finally {
+ fs.rmSync(extractDir, { recursive: true, force: true });
+ if (fs.existsSync(archivePath)) fs.unlinkSync(archivePath);
}
}
@@ -216,8 +260,9 @@ async function main() {
return;
}
+ const config = BINARIES[args.platformArch];
console.log(`Downloading for target platform (${args.platformArch}):`);
- const ok = await downloadBinary(args.platformArch, BINARIES[args.platformArch], args.isForce);
+ const ok = await downloadBinary(args.platformArch, config, args.isForce);
if (!ok) {
console.error(`Failed to download binaries for ${args.platformArch}`);
process.exitCode = 1;
@@ -235,16 +280,11 @@ async function main() {
}
if (args.shouldCleanup) {
- const wsPrefix = `sherpa-onnx-ws-${args.platformArch}`;
- const diarizePrefix = `sherpa-onnx-diarize-${args.platformArch}`;
- const files = fs.readdirSync(BIN_DIR).filter((f) => f.startsWith("sherpa-onnx"));
- files.forEach((file) => {
- if (!file.startsWith(wsPrefix) && !file.startsWith(diarizePrefix)) {
- const filePath = path.join(BIN_DIR, file);
- console.log(`Removing old binary: ${file}`);
- fs.unlinkSync(filePath);
- }
- });
+ cleanupFiles(BIN_DIR, "sherpa-onnx", [
+ `sherpa-onnx-ws-${args.platformArch}`,
+ `sherpa-onnx-online-ws-${args.platformArch}`,
+ `sherpa-onnx-diarize-${args.platformArch}`,
+ ]);
}
} else {
console.log("Downloading binaries for all platforms:");
diff --git a/scripts/lib/download-utils.js b/scripts/lib/download-utils.js
index 0c40f33b0..a3cc6e9f7 100644
--- a/scripts/lib/download-utils.js
+++ b/scripts/lib/download-utils.js
@@ -320,12 +320,13 @@ function setExecutable(filePath) {
}
function cleanupFiles(binDir, prefix, keepPrefix) {
+ const keepPrefixes = Array.isArray(keepPrefix) ? keepPrefix : [keepPrefix];
// Never delete shared libraries; only platform binaries. The b9763 split ships
// llama-server-impl.dll, which shares the "llama-server" prefix and must survive.
const isLibrary = (f) => /\.(dll|dylib)$/i.test(f) || /\.so(\.\d+)*$/.test(f);
const files = fs.readdirSync(binDir).filter((f) => f.startsWith(prefix));
files.forEach((file) => {
- if (!file.startsWith(keepPrefix) && !isLibrary(file)) {
+ if (!keepPrefixes.some((keep) => file.startsWith(keep)) && !isLibrary(file)) {
const filePath = path.join(binDir, file);
console.log(`Removing old binary: ${file}`);
fs.unlinkSync(filePath);
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 6150be7d1..cf0fb1d7b 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -5508,12 +5508,21 @@ class IPCHandlers {
let dictationPreviewLanguage = null;
let dictationPreviewSessionActive = false;
let dictationPreviewChunkCount = 0;
+ // Online-runtime models stream here instead of the 1.5s chunked path.
+ let dictationPreviewStream = null;
+ // Bumped on every reset so async preview work can detect a stale session.
+ let dictationPreviewGen = 0;
const resetDictationPreviewState = ({ preserveSession = false } = {}) => {
+ dictationPreviewGen++;
if (dictationPreviewTimer) {
clearInterval(dictationPreviewTimer);
dictationPreviewTimer = null;
}
+ if (dictationPreviewStream) {
+ dictationPreviewStream.abort();
+ dictationPreviewStream = null;
+ }
dictationPreviewMode = false;
if (!preserveSession) {
dictationPreviewSessionActive = false;
@@ -6208,6 +6217,7 @@ class IPCHandlers {
ipcMain.handle("start-dictation-preview", async (_event, { provider, model, language }) => {
resetDictationPreviewState();
+ const gen = dictationPreviewGen;
dictationPreviewMode = true;
dictationPreviewSessionActive = true;
dictationPreviewProvider = provider;
@@ -6215,6 +6225,35 @@ class IPCHandlers {
dictationPreviewLanguage = language || null;
dictationPreviewChunkCount = 0;
this.windowManager.showTranscriptionPreview("");
+
+ if (provider === "nvidia" && this.parakeetManager.supportsOnlineStreaming(model)) {
+ try {
+ const stream = await this.parakeetManager.createOnlineStream(model, {
+ onUpdate: (text) => {
+ if (gen === dictationPreviewGen && text) {
+ this.windowManager.showTranscriptionPreview(text);
+ }
+ },
+ });
+ if (gen !== dictationPreviewGen) {
+ stream.abort();
+ return { success: true };
+ }
+ dictationPreviewStream = stream;
+ for (const chunk of dictationPreviewBuffer) {
+ stream.sendPcm16(chunk);
+ }
+ dictationPreviewBuffer = [];
+ return { success: true };
+ } catch (error) {
+ debugLogger.warn("Online preview stream unavailable, falling back to chunked preview", {
+ model,
+ error: error.message,
+ });
+ }
+ }
+
+ if (gen !== dictationPreviewGen) return { success: true };
dictationPreviewTimer = setInterval(() => transcribeDictationPreviewChunk(), 1500);
return { success: true };
});
@@ -6229,9 +6268,12 @@ class IPCHandlers {
bufferSize: dictationPreviewBuffer.length,
});
}
- dictationPreviewBuffer.push(
- Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer)
- );
+ const pcm = Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer);
+ if (dictationPreviewStream) {
+ dictationPreviewStream.sendPcm16(pcm);
+ return;
+ }
+ dictationPreviewBuffer.push(pcm);
});
ipcMain.handle("dismiss-dictation-preview", async () => {
@@ -6272,7 +6314,16 @@ class IPCHandlers {
}
clearInterval(dictationPreviewTimer);
dictationPreviewTimer = null;
- await transcribeDictationPreviewChunk();
+ if (dictationPreviewStream) {
+ const stream = dictationPreviewStream;
+ dictationPreviewStream = null;
+ const { text } = await stream.finish().catch(() => ({ text: "" }));
+ if (text && dictationPreviewSessionActive) {
+ this.windowManager.showTranscriptionPreview(text);
+ }
+ } else {
+ await transcribeDictationPreviewChunk();
+ }
resetDictationPreviewState({ preserveSession: true });
if (!dictationPreviewSessionActive) {
return { success: true };
diff --git a/src/helpers/liveSpeakerIdentifier.js b/src/helpers/liveSpeakerIdentifier.js
index 5059c1d8d..d39dd620c 100644
--- a/src/helpers/liveSpeakerIdentifier.js
+++ b/src/helpers/liveSpeakerIdentifier.js
@@ -2,7 +2,7 @@ const fs = require("fs");
const debugLogger = require("./debugLogger");
const speakerEmbeddings = require("./speakerEmbeddings");
const { MAX_EMBEDDING_SECONDS } = speakerEmbeddings;
-const { downsample24kTo16k } = require("../utils/audioUtils");
+const { downsample24kTo16k, pcm16ToFloat32 } = require("../utils/audioUtils");
const { MAX_SPEAKER_COUNT } = require("../constants/speakerDetection.json");
function clampMaxSpeakers(value) {
@@ -31,17 +31,6 @@ const MATCH_MARGIN = 0.03;
const LIVE_WINDOW_PADDING_SECONDS = 0.75;
const DEFAULT_VAD_STATE_SHAPE = [2, 1, 64];
-function pcm16BufferToFloat32(buffer) {
- const input = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);
- const output = new Float32Array(input.length);
-
- for (let i = 0; i < input.length; i += 1) {
- output[i] = input[i] / 32768;
- }
-
- return output;
-}
-
function appendFloat32(existing, next) {
if (!existing.length) return next;
if (!next.length) return existing;
@@ -424,7 +413,7 @@ class LiveSpeakerIdentifier {
);
if (!downsampled.length) return;
- this.audioRemainder = appendFloat32(this.audioRemainder, pcm16BufferToFloat32(downsampled));
+ this.audioRemainder = appendFloat32(this.audioRemainder, pcm16ToFloat32(downsampled));
while (this.audioRemainder.length >= VAD_WINDOW_SIZE) {
const window = this.audioRemainder.subarray(0, VAD_WINDOW_SIZE);
diff --git a/src/helpers/meetingEchoLeakDetector.js b/src/helpers/meetingEchoLeakDetector.js
index 35500e09b..91b9c4459 100644
--- a/src/helpers/meetingEchoLeakDetector.js
+++ b/src/helpers/meetingEchoLeakDetector.js
@@ -30,16 +30,7 @@ const DOUBLE_TALK_EXPLAINED = 0.3;
const DOUBLE_TALK_RESIDUAL = 0.2;
const DOUBLE_TALK_MIC_TO_SYSTEM_RATIO = 1.16;
-function pcm16BufferToFloat32(buffer) {
- const input = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);
- const output = new Float32Array(input.length);
-
- for (let i = 0; i < input.length; i += 1) {
- output[i] = input[i] / 32768;
- }
-
- return output;
-}
+const { pcm16ToFloat32 } = require("../utils/audioUtils");
function computeRms(samples) {
if (!samples.length) {
@@ -70,7 +61,7 @@ class MeetingEchoLeakDetector {
return;
}
- const samples = pcm16BufferToFloat32(buffer);
+ const samples = pcm16ToFloat32(buffer);
if (!samples.length) {
return;
}
@@ -90,7 +81,7 @@ class MeetingEchoLeakDetector {
return null;
}
- const samples = pcm16BufferToFloat32(buffer);
+ const samples = pcm16ToFloat32(buffer);
if (!samples.length) {
return null;
}
diff --git a/src/helpers/parakeet.js b/src/helpers/parakeet.js
index e01a0c59b..3fe0f18f5 100644
--- a/src/helpers/parakeet.js
+++ b/src/helpers/parakeet.js
@@ -14,6 +14,7 @@ const ParakeetServerManager = require("./parakeetServer");
const { getModelsDirForService } = require("./modelDirUtils");
const modelRegistryData = require("../models/modelRegistryData.json");
+const { getModelRuntime, REQUIRED_MODEL_FILES } = require("./parakeetModelInfo");
function getParakeetModelConfig(modelName) {
const modelInfo = modelRegistryData.parakeetModels[modelName];
@@ -72,7 +73,7 @@ class ParakeetManager {
if (
localTranscriptionProvider === "nvidia" &&
parakeetModel &&
- this.serverManager.isAvailable()
+ this.serverManager.isAvailable(getModelRuntime(parakeetModel))
) {
if (this.serverManager.isModelDownloaded(parakeetModel)) {
debugLogger.info("Pre-warming parakeet server", { model: parakeetModel });
@@ -112,15 +113,16 @@ class ParakeetManager {
debugLogger.info("Parakeet initialization complete", {
totalTimeMs: Date.now() - startTime,
- binaryAvailable: this.serverManager.isAvailable(),
+ binaryAvailable: this.serverManager.hasAnyWsBinary(),
});
}
async logDependencyStatus() {
const status = {
sherpaOnnx: {
- available: this.serverManager.isAvailable(),
- path: this.serverManager.getBinaryPath(),
+ available: this.serverManager.hasAnyWsBinary(),
+ path:
+ this.serverManager.getBinaryPath("offline") || this.serverManager.getBinaryPath("online"),
},
models: [],
};
@@ -154,16 +156,13 @@ class ParakeetManager {
}
async checkInstallation() {
- const binaryPath = this.serverManager.getBinaryPath();
+ const binaryPath =
+ this.serverManager.getBinaryPath("offline") || this.serverManager.getBinaryPath("online");
if (!binaryPath) {
return { installed: false, working: false };
}
- return {
- installed: true,
- working: this.serverManager.isAvailable(),
- path: binaryPath,
- };
+ return { installed: true, working: true, path: binaryPath };
}
async startServer(modelName) {
@@ -179,22 +178,36 @@ class ParakeetManager {
return this.serverManager.getServerStatus();
}
+ supportsOnlineStreaming(modelName) {
+ return getModelRuntime(modelName) === "online";
+ }
+
+ async createOnlineStream(modelName, options = {}) {
+ this.validateModelName(modelName);
+ const started = await this.serverManager.startServer(modelName);
+ if (!started.success) {
+ throw new Error(started.reason || "Failed to start parakeet streaming server");
+ }
+ return this.serverManager.createOnlineStream(options);
+ }
+
async transcribeLocalParakeet(audioBlob, options = {}) {
+ const model = options.model || "parakeet-tdt-0.6b-v3";
+ const serverAvailable = this.serverManager.isAvailable(getModelRuntime(model));
+
debugLogger.logSTTPipeline("transcribeLocalParakeet - start", {
options,
audioBlobType: audioBlob?.constructor?.name,
audioBlobSize: audioBlob?.byteLength || audioBlob?.size || 0,
- serverAvailable: this.serverManager.isAvailable(),
+ serverAvailable,
});
- if (!this.serverManager.isAvailable()) {
+ if (!serverAvailable) {
throw new Error(
"sherpa-onnx binary not found. Please ensure the app is installed correctly."
);
}
- const model = options.model || "parakeet-tdt-0.6b-v3";
-
if (!this.serverManager.isModelDownloaded(model)) {
throw new Error(
`Parakeet model "${model}" not downloaded. Please download it from Settings.`
@@ -341,7 +354,7 @@ class ParakeetManager {
progressCallback({ type: "complete", model: modelName, percentage: 100 });
}
- if (this.serverManager.isAvailable()) {
+ if (this.serverManager.isAvailable(getModelRuntime(modelName))) {
this.serverManager.startServer(modelName).catch((err) => {
debugLogger.warn("Post-download server pre-warm failed (non-fatal)", {
error: err.message,
@@ -392,7 +405,10 @@ class ParakeetManager {
for (const entry of entries) {
const entryPath = path.join(extractDir, entry);
const stat = await fsPromises.stat(entryPath);
- if (stat.isDirectory() && entry.includes("parakeet")) {
+ if (
+ stat.isDirectory() &&
+ REQUIRED_MODEL_FILES.every((file) => fs.existsSync(path.join(entryPath, file)))
+ ) {
modelDir = entry;
break;
}
@@ -411,13 +427,7 @@ class ParakeetManager {
}
}
- const requiredFiles = [
- "encoder.int8.onnx",
- "decoder.int8.onnx",
- "joiner.int8.onnx",
- "tokens.txt",
- ];
- const missing = requiredFiles.filter((f) => !fs.existsSync(path.join(targetDir, f)));
+ const missing = REQUIRED_MODEL_FILES.filter((f) => !fs.existsSync(path.join(targetDir, f)));
if (missing.length > 0) {
throw new Error(`Extracted model is missing required files: ${missing.join(", ")}`);
}
@@ -600,8 +610,8 @@ class ParakeetManager {
modelsDir: this.getModelsDir(),
models: [],
};
-
- const binaryPath = this.serverManager.getBinaryPath();
+ const binaryPath =
+ this.serverManager.getBinaryPath("offline") || this.serverManager.getBinaryPath("online");
if (binaryPath) {
diagnostics.sherpaOnnx = { available: true, path: binaryPath };
}
diff --git a/src/helpers/parakeetModelInfo.js b/src/helpers/parakeetModelInfo.js
new file mode 100644
index 000000000..abea60fd5
--- /dev/null
+++ b/src/helpers/parakeetModelInfo.js
@@ -0,0 +1,14 @@
+const modelRegistryData = require("../models/modelRegistryData.json");
+
+const REQUIRED_MODEL_FILES = [
+ "encoder.int8.onnx",
+ "decoder.int8.onnx",
+ "joiner.int8.onnx",
+ "tokens.txt",
+];
+
+function getModelRuntime(modelName) {
+ return modelRegistryData.parakeetModels?.[modelName]?.runtime === "online" ? "online" : "offline";
+}
+
+module.exports = { REQUIRED_MODEL_FILES, getModelRuntime };
diff --git a/src/helpers/parakeetServer.js b/src/helpers/parakeetServer.js
index b390f5e4d..4d3d8be16 100644
--- a/src/helpers/parakeetServer.js
+++ b/src/helpers/parakeetServer.js
@@ -11,6 +11,7 @@ const {
} = require("./ffmpegUtils");
const { getSafeTempDir } = require("./safeTempDir");
const ParakeetWsServer = require("./parakeetWsServer");
+const { getModelRuntime, REQUIRED_MODEL_FILES } = require("./parakeetModelInfo");
const SAMPLE_RATE = 16000;
const BYTES_PER_SAMPLE = 4; // float32
@@ -23,12 +24,16 @@ class ParakeetServerManager {
this.wsServer = new ParakeetWsServer();
}
- getBinaryPath() {
- return this.wsServer.getWsBinaryPath();
+ getBinaryPath(runtime) {
+ return this.wsServer.getWsBinaryPath(runtime);
}
- isAvailable() {
- return this.wsServer.isAvailable();
+ isAvailable(runtime) {
+ return this.wsServer.isAvailable(runtime);
+ }
+
+ hasAnyWsBinary() {
+ return this.wsServer.hasAnyWsBinary();
}
getModelsDir() {
@@ -37,22 +42,9 @@ class ParakeetServerManager {
isModelDownloaded(modelName) {
const modelDir = path.join(this.getModelsDir(), modelName);
- const requiredFiles = [
- "encoder.int8.onnx",
- "decoder.int8.onnx",
- "joiner.int8.onnx",
- "tokens.txt",
- ];
-
if (!fs.existsSync(modelDir)) return false;
- for (const file of requiredFiles) {
- if (!fs.existsSync(path.join(modelDir, file))) {
- return false;
- }
- }
-
- return true;
+ return REQUIRED_MODEL_FILES.every((file) => fs.existsSync(path.join(modelDir, file)));
}
async _ensureWav(audioBuffer) {
@@ -99,7 +91,7 @@ class ParakeetServerManager {
const { wavBuffer, filesToCleanup } = await this._ensureWav(audioBuffer);
try {
if (!this.wsServer.ready || this.wsServer.modelName !== modelName) {
- await this.wsServer.start(modelName, modelDir);
+ await this.wsServer.start(modelName, modelDir, getModelRuntime(modelName));
}
const samples = wavToFloat32Samples(wavBuffer);
@@ -168,7 +160,8 @@ class ParakeetServerManager {
}
async startServer(modelName) {
- if (!this.wsServer.isAvailable()) {
+ const runtime = getModelRuntime(modelName);
+ if (!this.wsServer.isAvailable(runtime)) {
return { success: false, reason: "parakeet WS server binary not found" };
}
@@ -178,7 +171,7 @@ class ParakeetServerManager {
}
try {
- await this.wsServer.start(modelName, modelDir);
+ await this.wsServer.start(modelName, modelDir, runtime);
return { success: true, port: this.wsServer.port };
} catch (error) {
debugLogger.error("Failed to start parakeet WS server", { error: error.message });
@@ -194,12 +187,8 @@ class ParakeetServerManager {
return this.wsServer.getStatus();
}
- getStatus() {
- return {
- available: this.isAvailable(),
- binaryPath: this.getBinaryPath(),
- modelsDir: this.getModelsDir(),
- };
+ createOnlineStream(options) {
+ return this.wsServer.createOnlineStream(options);
}
}
diff --git a/src/helpers/parakeetWsResult.js b/src/helpers/parakeetWsResult.js
new file mode 100644
index 000000000..a916ac812
--- /dev/null
+++ b/src/helpers/parakeetWsResult.js
@@ -0,0 +1,59 @@
+function parseOfflineMessage(message) {
+ const text = String(message || "").trim();
+ try {
+ const parsed = JSON.parse(text);
+ return typeof parsed?.text === "string" ? parsed.text.trim() : text;
+ } catch {
+ return text;
+ }
+}
+
+// Dedupes finalized segments by id; text() is finalized text plus the trailing partial.
+function createOnlineAccumulator() {
+ const finalizedSegments = new Set();
+ let finalizedText = "";
+ let partialText = "";
+ let fallbackKey = 0;
+
+ const text = () =>
+ finalizedText && partialText ? `${finalizedText} ${partialText}` : finalizedText || partialText;
+
+ return {
+ push(message) {
+ let parsed;
+ try {
+ parsed = JSON.parse(message);
+ } catch {
+ parsed = { text: message };
+ }
+ if (!parsed || typeof parsed !== "object") return text();
+
+ const messageText = String(parsed.text ?? "").trim();
+ if (!messageText) return text();
+
+ if (!parsed.is_final) {
+ partialText = finalizedSegments.has(parsed.segment) ? "" : messageText;
+ return text();
+ }
+
+ const segment = parsed.segment ?? `fallback:${fallbackKey++}`;
+ if (!finalizedSegments.has(segment)) {
+ finalizedSegments.add(segment);
+ finalizedText = finalizedText ? `${finalizedText} ${messageText}` : messageText;
+ }
+ partialText = "";
+ return text();
+ },
+ text,
+ };
+}
+
+function parseOnlineMessages(messages) {
+ const accumulator = createOnlineAccumulator();
+ for (const message of messages) {
+ accumulator.push(message);
+ }
+ return accumulator.text();
+}
+
+module.exports = { parseOfflineMessage, parseOnlineMessages, createOnlineAccumulator };
diff --git a/src/helpers/parakeetWsServer.js b/src/helpers/parakeetWsServer.js
index b6366ace4..2a0b8d243 100644
--- a/src/helpers/parakeetWsServer.js
+++ b/src/helpers/parakeetWsServer.js
@@ -11,12 +11,16 @@ const {
} = require("../utils/serverUtils");
const { getSafeTempDir } = require("./safeTempDir");
const sidecarPidFile = require("./sidecarPidFile");
+const { parseOfflineMessage, createOnlineAccumulator } = require("./parakeetWsResult");
+const { pcm16ToFloat32 } = require("../utils/audioUtils");
const PORT_RANGE_START = 6006;
const PORT_RANGE_END = 6029;
const STARTUP_TIMEOUT_MS = 60000;
const HEALTH_CHECK_INTERVAL_MS = 5000;
const TRANSCRIPTION_TIMEOUT_MS = 300000;
+const ONLINE_CHUNK_BYTES = 8000 * 4;
+const ONLINE_FINISH_TIMEOUT_MS = 10000;
class ParakeetWsServer {
constructor() {
@@ -25,35 +29,39 @@ class ParakeetWsServer {
this.ready = false;
this.modelName = null;
this.modelDir = null;
+ this.modelRuntime = "offline";
this.startupPromise = null;
this.healthCheckInterval = null;
- this.cachedWsBinaryPath = null;
+ this.cachedBinaryPaths = {};
}
- getWsBinaryPath() {
- if (this.cachedWsBinaryPath) return this.cachedWsBinaryPath;
+ getWsBinaryPath(runtime = "offline") {
+ if (this.cachedBinaryPaths[runtime]) return this.cachedBinaryPaths[runtime];
const platformArch = `${process.platform}-${process.arch}`;
+ const prefix = runtime === "online" ? "sherpa-onnx-online-ws" : "sherpa-onnx-ws";
const binaryName =
- process.platform === "win32"
- ? `sherpa-onnx-ws-${platformArch}.exe`
- : `sherpa-onnx-ws-${platformArch}`;
+ process.platform === "win32" ? `${prefix}-${platformArch}.exe` : `${prefix}-${platformArch}`;
const resolved = resolveBinaryPath(binaryName);
- if (resolved) this.cachedWsBinaryPath = resolved;
+ if (resolved) this.cachedBinaryPaths[runtime] = resolved;
return resolved;
}
- isAvailable() {
- return this.getWsBinaryPath() !== null;
+ isAvailable(runtime = "offline") {
+ return this.getWsBinaryPath(runtime) !== null;
}
- async start(modelName, modelDir) {
+ hasAnyWsBinary() {
+ return this.isAvailable("offline") || this.isAvailable("online");
+ }
+
+ async start(modelName, modelDir, runtime = "offline") {
if (this.startupPromise) return this.startupPromise;
if (this.ready && this.modelName === modelName) return;
if (this.process) await this.stop();
- this.startupPromise = this._doStart(modelName, modelDir);
+ this.startupPromise = this._doStart(modelName, modelDir, runtime);
try {
await this.startupPromise;
} finally {
@@ -61,25 +69,29 @@ class ParakeetWsServer {
}
}
- async _doStart(modelName, modelDir) {
- const wsBinary = this.getWsBinaryPath();
- if (!wsBinary) throw new Error("sherpa-onnx WS server binary not found");
+ async _doStart(modelName, modelDir, runtime) {
+ const wsBinary = this.getWsBinaryPath(runtime);
+ if (!wsBinary) throw new Error(`sherpa-onnx ${runtime} WS server binary not found`);
if (!fs.existsSync(modelDir)) throw new Error(`Model directory not found: ${modelDir}`);
this.port = await findAvailablePort(PORT_RANGE_START, PORT_RANGE_END);
this.modelName = modelName;
this.modelDir = modelDir;
+ this.modelRuntime = runtime;
+ const threads = Math.max(1, Math.min(4, Math.floor(os.cpus().length * 0.75)));
const args = [
`--tokens=${path.join(modelDir, "tokens.txt")}`,
`--encoder=${path.join(modelDir, "encoder.int8.onnx")}`,
`--decoder=${path.join(modelDir, "decoder.int8.onnx")}`,
`--joiner=${path.join(modelDir, "joiner.int8.onnx")}`,
`--port=${this.port}`,
- `--num-threads=${Math.max(1, Math.min(4, Math.floor(os.cpus().length * 0.75)))}`,
+ ...(runtime === "online"
+ ? [`--num-work-threads=${threads}`, "--warm-up=0"]
+ : [`--num-threads=${threads}`]),
];
- debugLogger.debug("Starting parakeet WS server", { port: this.port, modelName, args });
+ debugLogger.debug("Starting parakeet WS server", { port: this.port, modelName, runtime, args });
this.process = spawn(wsBinary, args, {
stdio: ["ignore", "pipe", "pipe"],
@@ -130,6 +142,7 @@ class ParakeetWsServer {
debugLogger.info("parakeet-ws server started successfully", {
port: this.port,
model: modelName,
+ runtime,
});
await this._warmUp();
@@ -210,6 +223,14 @@ class ParakeetWsServer {
throw new Error("parakeet-ws server is not running");
}
+ if (this.modelRuntime === "online") {
+ return this._transcribeOnline(samplesBuffer);
+ }
+
+ return this._transcribeOffline(samplesBuffer, sampleRate);
+ }
+
+ _transcribeOffline(samplesBuffer, sampleRate) {
return new Promise((resolve, reject) => {
const startTime = Date.now();
let result = "";
@@ -259,12 +280,7 @@ class ParakeetWsServer {
resultPreview: result.slice(0, 200),
});
- try {
- const parsed = JSON.parse(result);
- resolve({ text: (parsed.text || "").trim(), elapsed });
- } catch {
- resolve({ text: result.trim(), elapsed });
- }
+ resolve({ text: parseOfflineMessage(result), elapsed });
});
ws.on("error", (error) => {
@@ -274,6 +290,142 @@ class ParakeetWsServer {
});
}
+ // samplesBuffer must already be 16kHz float32.
+ async _transcribeOnline(samplesBuffer) {
+ const startTime = Date.now();
+ let streamError = null;
+ let timedOut = false;
+
+ const stream = this.createOnlineStream({
+ onError: (error) => {
+ streamError = error;
+ },
+ });
+
+ debugLogger.debug("parakeet-ws sending streaming audio", {
+ samplesBytes: samplesBuffer.length,
+ });
+ for (let offset = 0; offset < samplesBuffer.length; offset += ONLINE_CHUNK_BYTES) {
+ stream.sendFloat32(samplesBuffer.subarray(offset, offset + ONLINE_CHUNK_BYTES));
+ }
+
+ const timeout = setTimeout(() => {
+ timedOut = true;
+ stream.abort();
+ }, TRANSCRIPTION_TIMEOUT_MS);
+ try {
+ const { text } = await stream.finish();
+ if (timedOut) throw new Error("parakeet-ws transcription timed out");
+ if (streamError) {
+ throw new Error(`parakeet-ws transcription failed: ${streamError.message}`);
+ }
+
+ const elapsed = Date.now() - startTime;
+ debugLogger.debug("parakeet-ws streaming transcription completed", {
+ elapsed,
+ resultLength: text.length,
+ resultPreview: text.slice(0, 200),
+ });
+ return { text, elapsed };
+ } finally {
+ clearTimeout(timeout);
+ }
+ }
+
+ createOnlineStream({ onUpdate, onError } = {}) {
+ if (!this.ready || !this.process) {
+ throw new Error("parakeet-ws server is not running");
+ }
+ if (this.modelRuntime !== "online") {
+ throw new Error("createOnlineStream requires an online-runtime model");
+ }
+
+ const results = createOnlineAccumulator();
+ const pendingChunks = [];
+ let finishResolve = null;
+ let closed = false;
+ let lastEmitted = "";
+
+ const ws = new WebSocket(`ws://127.0.0.1:${this.port}`);
+
+ const settle = () => {
+ if (closed) return;
+ closed = true;
+ if (finishResolve) finishResolve({ text: results.text() });
+ };
+
+ const sendFloat32 = (float32Samples) => {
+ if (closed || finishResolve) return;
+ if (ws.readyState === WebSocket.OPEN) {
+ ws.send(float32Samples, (err) => {
+ if (err) {
+ debugLogger.error("parakeet-ws online stream send error", { error: err.message });
+ }
+ });
+ } else {
+ pendingChunks.push(float32Samples);
+ }
+ };
+
+ ws.on("open", () => {
+ for (const chunk of pendingChunks) {
+ ws.send(chunk);
+ }
+ pendingChunks.length = 0;
+ if (finishResolve) ws.send("Done");
+ });
+
+ ws.on("message", (data) => {
+ const message = data.toString();
+ if (message === "Done!") {
+ ws.close();
+ return;
+ }
+ const text = results.push(message);
+ if (!closed && text && text !== lastEmitted) {
+ lastEmitted = text;
+ onUpdate?.(text);
+ }
+ });
+
+ ws.on("close", () => settle());
+
+ ws.on("error", (error) => {
+ debugLogger.warn("parakeet-ws online stream error", { error: error.message });
+ onError?.(error);
+ settle();
+ });
+
+ return {
+ sendFloat32,
+ sendPcm16: (pcmBuffer) => sendFloat32(pcm16ToFloat32(pcmBuffer)),
+ finish: () =>
+ new Promise((resolve) => {
+ if (closed) {
+ resolve({ text: results.text() });
+ return;
+ }
+ finishResolve = resolve;
+ if (ws.readyState === WebSocket.OPEN) ws.send("Done");
+ // Backstop in case the server never acknowledges with "Done!".
+ setTimeout(() => {
+ if (!closed) {
+ try {
+ ws.close();
+ } catch {}
+ settle();
+ }
+ }, ONLINE_FINISH_TIMEOUT_MS).unref?.();
+ }),
+ abort: () => {
+ try {
+ ws.close();
+ } catch {}
+ settle();
+ },
+ };
+ }
+
async stop() {
this.stopHealthCheck();
@@ -295,11 +447,12 @@ class ParakeetWsServer {
this.port = null;
this.modelName = null;
this.modelDir = null;
+ this.modelRuntime = "offline";
}
getStatus() {
return {
- available: this.isAvailable(),
+ available: this.hasAnyWsBinary(),
running: this.ready && this.process !== null,
port: this.port,
modelName: this.modelName,
diff --git a/src/helpers/sidecarReaper.js b/src/helpers/sidecarReaper.js
index 6e9633b38..7110016bd 100644
--- a/src/helpers/sidecarReaper.js
+++ b/src/helpers/sidecarReaper.js
@@ -3,11 +3,11 @@ const debugLogger = require("./debugLogger");
const sidecarPidFile = require("./sidecarPidFile");
const EXPECTED_BINARY_FRAGMENTS = {
- parakeet: "sherpa-onnx-ws",
- whisper: "whisper-server",
- llama: "llama-server",
- qdrant: "qdrant",
- diarization: "sherpa-onnx-diarize",
+ parakeet: ["sherpa-onnx-ws-", "sherpa-onnx-online-ws-"],
+ whisper: ["whisper-server"],
+ llama: ["llama-server"],
+ qdrant: ["qdrant"],
+ diarization: ["sherpa-onnx-diarize"],
};
function isProcessAlive(pid) {
@@ -42,8 +42,13 @@ function processCommand(pid) {
function reapStaleSidecars() {
const entries = sidecarPidFile.readAll();
for (const { name, pid } of entries) {
- const fragment = EXPECTED_BINARY_FRAGMENTS[name];
- if (fragment && isProcessAlive(pid) && processCommand(pid).includes(fragment)) {
+ const fragments = EXPECTED_BINARY_FRAGMENTS[name];
+ if (fragments && isProcessAlive(pid)) {
+ const command = processCommand(pid);
+ if (!fragments.some((fragment) => command.includes(fragment))) {
+ sidecarPidFile.clear(name);
+ continue;
+ }
debugLogger.warn("Reaping stale sidecar", { name, pid });
try {
process.kill(pid, "SIGTERM");
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index f40db87fc..100c2994e 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -1878,7 +1878,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "Schnelle mehrsprachige ASR mit automatischer Spracherkennung (25 Sprachen)",
- "parakeet_unified_en_0_6b": "Englisch-only ASR mit Spitzen-Genauigkeit (5,91% durchschnittliche WER)"
+ "parakeet_unified_en_0_6b": "Englisch-only ASR mit Spitzen-Genauigkeit (5,91% durchschnittliche WER)",
+ "nemotron_speech_streaming_en_0_6b": "Englische Streaming-ASR von NVIDIA Nemotron",
+ "nemotron_3_5_asr_streaming_0_6b": "Mehrsprachige Streaming-ASR von NVIDIA Nemotron mit automatischer Spracherkennung (15 Sprachen)"
},
"whisper": {
"tiny": "Am schnellsten, geringere Qualität",
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index 5cee307f0..63b424acb 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -1966,7 +1966,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "Fast multilingual ASR with auto language detection (25 languages)",
- "parakeet_unified_en_0_6b": "English-only ASR with state-of-the-art accuracy (5.91% avg WER)"
+ "parakeet_unified_en_0_6b": "English-only ASR with state-of-the-art accuracy (5.91% avg WER)",
+ "nemotron_speech_streaming_en_0_6b": "English streaming ASR from NVIDIA Nemotron",
+ "nemotron_3_5_asr_streaming_0_6b": "Multilingual streaming ASR from NVIDIA Nemotron with auto language detection (15 languages)"
},
"diarization": {
"segmentation": "Speech boundary detection for speaker diarization",
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 8448c327c..a29a39f96 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -1926,7 +1926,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "ASR multilingüe rápido con detección automática de idioma (25 idiomas)",
- "parakeet_unified_en_0_6b": "ASR solo en inglés con precisión de vanguardia (5,91% WER promedio)"
+ "parakeet_unified_en_0_6b": "ASR solo en inglés con precisión de vanguardia (5,91% WER promedio)",
+ "nemotron_speech_streaming_en_0_6b": "ASR en inglés por streaming de NVIDIA Nemotron",
+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilingüe por streaming de NVIDIA Nemotron con detección automática de idioma (15 idiomas)"
},
"whisper": {
"tiny": "El más rápido, menor calidad",
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index dfbcbea23..e7238fe82 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -1926,7 +1926,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "ASR multilingue rapide avec détection automatique de langue (25 langues)",
- "parakeet_unified_en_0_6b": "ASR anglais uniquement avec précision de pointe (5,91 % WER moyen)"
+ "parakeet_unified_en_0_6b": "ASR anglais uniquement avec précision de pointe (5,91 % WER moyen)",
+ "nemotron_speech_streaming_en_0_6b": "ASR anglais en streaming de NVIDIA Nemotron",
+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilingue en streaming de NVIDIA Nemotron avec détection automatique de langue (15 langues)"
},
"whisper": {
"tiny": "Le plus rapide, qualité plus faible",
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index 2683b2dc2..382253a1e 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -1878,7 +1878,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "ASR multilingue veloce con rilevamento automatico della lingua (25 lingue)",
- "parakeet_unified_en_0_6b": "ASR solo inglese con precisione all'avanguardia (5,91% WER medio)"
+ "parakeet_unified_en_0_6b": "ASR solo inglese con precisione all'avanguardia (5,91% WER medio)",
+ "nemotron_speech_streaming_en_0_6b": "ASR streaming in inglese di NVIDIA Nemotron",
+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilingue in streaming di NVIDIA Nemotron con rilevamento automatico della lingua (15 lingue)"
},
"whisper": {
"tiny": "Il più veloce, qualità inferiore",
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index ae90fe29d..c21cac917 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -1878,7 +1878,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "高速多言語 ASR、自動言語検出対応(25 言語)",
- "parakeet_unified_en_0_6b": "英語専用の最先端精度ASR(平均WER 5.91%)"
+ "parakeet_unified_en_0_6b": "英語専用の最先端精度ASR(平均WER 5.91%)",
+ "nemotron_speech_streaming_en_0_6b": "NVIDIA Nemotron の英語ストリーミング ASR",
+ "nemotron_3_5_asr_streaming_0_6b": "NVIDIA Nemotron の多言語ストリーミング ASR、自動言語検出対応(15 言語)"
},
"whisper": {
"tiny": "最速、品質は低め",
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index 1a64da157..7b66b6b59 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -1857,7 +1857,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "ASR multilíngue rápido com detecção automática de idioma (25 idiomas)",
- "parakeet_unified_en_0_6b": "ASR apenas em inglês com precisão de ponta (5,91% WER médio)"
+ "parakeet_unified_en_0_6b": "ASR apenas em inglês com precisão de ponta (5,91% WER médio)",
+ "nemotron_speech_streaming_en_0_6b": "ASR em inglês por streaming da NVIDIA Nemotron",
+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilíngue por streaming da NVIDIA Nemotron com detecção automática de idioma (15 idiomas)"
},
"whisper": {
"tiny": "Mais rápido, qualidade inferior",
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index a5fa12f14..62d7d09f4 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -1878,7 +1878,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "Быстрое многоязычное распознавание с автоопределением языка (25 языков)",
- "parakeet_unified_en_0_6b": "ASR только для английского с передовой точностью (средний WER 5,91%)"
+ "parakeet_unified_en_0_6b": "ASR только для английского с передовой точностью (средний WER 5,91%)",
+ "nemotron_speech_streaming_en_0_6b": "Потоковое распознавание английской речи от NVIDIA Nemotron",
+ "nemotron_3_5_asr_streaming_0_6b": "Многоязычное потоковое распознавание речи от NVIDIA Nemotron с автоопределением языка (15 языков)"
},
"whisper": {
"tiny": "Самый быстрый, низкое качество",
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index 09393e2d5..94b2dfb2f 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -1873,7 +1873,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "快速多语言语音识别,自动语言检测(25 种语言)",
- "parakeet_unified_en_0_6b": "仅英语 ASR,最先进的准确度(平均 WER 5.91%)"
+ "parakeet_unified_en_0_6b": "仅英语 ASR,最先进的准确度(平均 WER 5.91%)",
+ "nemotron_speech_streaming_en_0_6b": "NVIDIA Nemotron 英语流式语音识别",
+ "nemotron_3_5_asr_streaming_0_6b": "NVIDIA Nemotron 多语言流式语音识别,自动语言检测(15 种语言)"
},
"whisper": {
"tiny": "最快速度,较低质量",
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index ab1492246..48f0d7ab3 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -1873,7 +1873,9 @@
"descriptions": {
"parakeet": {
"parakeet_tdt_0_6b_v3": "快速多語言 ASR,自動語言偵測(25 種語言)",
- "parakeet_unified_en_0_6b": "僅英語 ASR,最先進的準確度(平均 WER 5.91%)"
+ "parakeet_unified_en_0_6b": "僅英語 ASR,最先進的準確度(平均 WER 5.91%)",
+ "nemotron_speech_streaming_en_0_6b": "NVIDIA Nemotron 英語串流語音辨識",
+ "nemotron_3_5_asr_streaming_0_6b": "NVIDIA Nemotron 多語言串流語音辨識,自動語言偵測(15 種語言)"
},
"whisper": {
"tiny": "最快速,品質較低",
diff --git a/src/models/ModelRegistry.ts b/src/models/ModelRegistry.ts
index f6639c48f..9745b8485 100644
--- a/src/models/ModelRegistry.ts
+++ b/src/models/ModelRegistry.ts
@@ -99,6 +99,7 @@ export interface ParakeetModelInfo {
sizeMb: number;
language: string;
supportedLanguages: string[];
+ runtime?: "offline" | "online";
recommended?: boolean;
downloadUrl: string;
extractDir: string;
diff --git a/src/models/modelRegistryData.json b/src/models/modelRegistryData.json
index 3d75a1e8a..4620f7224 100644
--- a/src/models/modelRegistryData.json
+++ b/src/models/modelRegistryData.json
@@ -47,6 +47,46 @@
"downloadUrl": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-non-streaming.tar.bz2",
"extractDir": "sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-non-streaming",
"descriptionKey": "models.descriptions.parakeet.parakeet_unified_en_0_6b"
+ },
+ "nemotron-speech-streaming-en-0.6b": {
+ "name": "Nemotron Speech Streaming EN 0.6B",
+ "description": "English streaming ASR from NVIDIA Nemotron",
+ "size": "632MB",
+ "sizeMb": 632,
+ "language": "en",
+ "supportedLanguages": ["en"],
+ "runtime": "online",
+ "downloadUrl": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemotron-speech-streaming-en-0.6b-560ms-int8-2026-04-25.tar.bz2",
+ "extractDir": "sherpa-onnx-nemotron-speech-streaming-en-0.6b-560ms-int8-2026-04-25",
+ "descriptionKey": "models.descriptions.parakeet.nemotron_speech_streaming_en_0_6b"
+ },
+ "nemotron-3.5-asr-streaming-0.6b": {
+ "name": "Nemotron 3.5 ASR Streaming 0.6B",
+ "description": "Multilingual streaming ASR from NVIDIA Nemotron with auto language detection (15 languages)",
+ "size": "650MB",
+ "sizeMb": 650,
+ "language": "multilingual",
+ "supportedLanguages": [
+ "en",
+ "es",
+ "fr",
+ "it",
+ "pt",
+ "nl",
+ "de",
+ "tr",
+ "ru",
+ "ar",
+ "hi",
+ "ja",
+ "ko",
+ "vi",
+ "uk"
+ ],
+ "runtime": "online",
+ "downloadUrl": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11.tar.bz2",
+ "extractDir": "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11",
+ "descriptionKey": "models.descriptions.parakeet.nemotron_3_5_asr_streaming_0_6b"
}
},
"diarizationModels": {
diff --git a/src/utils/audioUtils.js b/src/utils/audioUtils.js
index fb7d751be..b7c4677ae 100644
--- a/src/utils/audioUtils.js
+++ b/src/utils/audioUtils.js
@@ -35,7 +35,17 @@ function pcm16ToWav(pcmBuffer, sampleRate = 16000, channels = 1) {
return Buffer.concat([header, pcmBuffer]);
}
+function pcm16ToFloat32(pcmBuffer) {
+ const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);
+ const output = new Float32Array(input.length);
+ for (let i = 0; i < input.length; i++) {
+ output[i] = input[i] / 32768;
+ }
+ return output;
+}
+
module.exports = {
downsample24kTo16k,
pcm16ToWav,
+ pcm16ToFloat32,
};
diff --git a/test/helpers/parakeetModelRegistry.test.js b/test/helpers/parakeetModelRegistry.test.js
new file mode 100644
index 000000000..7c5dff655
--- /dev/null
+++ b/test/helpers/parakeetModelRegistry.test.js
@@ -0,0 +1,44 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const modelData = require("../../src/models/modelRegistryData.json");
+const { BINARIES } = require("../../scripts/download-sherpa-onnx");
+
+test("Nemotron sherpa model uses the streaming runtime and bundled online server", () => {
+ const model = modelData.parakeetModels["nemotron-speech-streaming-en-0.6b"];
+
+ assert.equal(model.runtime, "online");
+ assert.equal(
+ model.extractDir,
+ "sherpa-onnx-nemotron-speech-streaming-en-0.6b-560ms-int8-2026-04-25"
+ );
+ assert.match(
+ model.downloadUrl,
+ /sherpa-onnx-nemotron-speech-streaming-en-0\.6b-560ms-int8-2026-04-25\.tar\.bz2$/
+ );
+ assert.equal(model.language, "en");
+ assert.deepEqual(model.supportedLanguages, ["en"]);
+
+ for (const [platformArch, config] of Object.entries(BINARIES)) {
+ assert.match(config.onlineBinaryPath, /sherpa-onnx-online-websocket-server/);
+ assert.match(config.onlineOutputName, new RegExp(`^sherpa-onnx-online-ws-${platformArch}`));
+ }
+});
+
+test("Nemotron 3.5 multilingual sherpa model uses the streaming runtime", () => {
+ const model = modelData.parakeetModels["nemotron-3.5-asr-streaming-0.6b"];
+
+ assert.equal(model.runtime, "online");
+ assert.equal(
+ model.extractDir,
+ "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11"
+ );
+ assert.match(
+ model.downloadUrl,
+ /sherpa-onnx-nemotron-3\.5-asr-streaming-0\.6b-560ms-int8-2026-06-11\.tar\.bz2$/
+ );
+ assert.equal(model.language, "multilingual");
+ assert.ok(model.supportedLanguages.length >= 15);
+ assert.ok(model.supportedLanguages.includes("en"));
+ assert.ok(model.supportedLanguages.includes("ja"));
+});
diff --git a/test/helpers/parakeetOnlineStream.test.js b/test/helpers/parakeetOnlineStream.test.js
new file mode 100644
index 000000000..18d02ba6e
--- /dev/null
+++ b/test/helpers/parakeetOnlineStream.test.js
@@ -0,0 +1,146 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { once } = require("node:events");
+
+const { WebSocketServer } = require("ws");
+
+const ParakeetWsServer = require("../../src/helpers/parakeetWsServer");
+const { pcm16ToFloat32 } = require("../../src/utils/audioUtils");
+
+// Mock sherpa online WS protocol: float32 binary frames in, JSON results out, "Done"/"Done!" handshake.
+async function startMockOnlineServer({ onBinary, finalSegment = 0 }) {
+ const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" });
+ await once(wss, "listening");
+
+ wss.on("connection", (socket) => {
+ let binaryFrames = 0;
+ socket.on("message", (data, isBinary) => {
+ if (isBinary) {
+ binaryFrames += 1;
+ onBinary?.(socket, data, binaryFrames);
+ return;
+ }
+ if (data.toString() === "Done") {
+ socket.send(
+ JSON.stringify({
+ text: `final after ${binaryFrames} frames`,
+ segment: finalSegment,
+ is_final: true,
+ })
+ );
+ socket.send("Done!");
+ }
+ });
+ });
+
+ return {
+ port: wss.address().port,
+ close: () => new Promise((resolve) => wss.close(resolve)),
+ };
+}
+
+function onlineWsServerAt(port) {
+ const server = new ParakeetWsServer();
+ server.ready = true;
+ server.process = { pid: 1 };
+ server.port = port;
+ server.modelRuntime = "online";
+ return server;
+}
+
+test("online stream emits live updates and finish resolves with the final text", async () => {
+ const mock = await startMockOnlineServer({
+ onBinary: (socket, _data, frameCount) => {
+ socket.send(
+ JSON.stringify({ text: `partial ${frameCount}`, segment: 0, is_final: false })
+ );
+ },
+ });
+
+ try {
+ const updates = [];
+ const stream = onlineWsServerAt(mock.port).createOnlineStream({
+ onUpdate: (text) => updates.push(text),
+ });
+
+ // Chunks sent before the socket opens must be queued, not dropped.
+ stream.sendPcm16(Buffer.alloc(3200));
+ stream.sendFloat32(pcm16ToFloat32(Buffer.alloc(3200)));
+
+ const { text } = await stream.finish();
+
+ assert.equal(text, "final after 2 frames");
+ assert.ok(updates.includes("partial 1"));
+ assert.ok(updates.includes("partial 2"));
+ } finally {
+ await mock.close();
+ }
+});
+
+test("finalized segments accumulate across endpoints in live updates", async () => {
+ let segment = 0;
+ const mock = await startMockOnlineServer({
+ finalSegment: 2,
+ onBinary: (socket, _data, frameCount) => {
+ // Simulate endpointing: every frame finalizes a segment.
+ socket.send(
+ JSON.stringify({ text: `segment ${frameCount}`, segment: segment++, is_final: true })
+ );
+ },
+ });
+
+ try {
+ let lastUpdate = "";
+ const stream = onlineWsServerAt(mock.port).createOnlineStream({
+ onUpdate: (text) => {
+ lastUpdate = text;
+ },
+ });
+
+ stream.sendPcm16(Buffer.alloc(3200));
+ stream.sendPcm16(Buffer.alloc(3200));
+
+ const { text } = await stream.finish();
+
+ assert.equal(text, "segment 1 segment 2 final after 2 frames");
+ assert.equal(lastUpdate, text);
+ } finally {
+ await mock.close();
+ }
+});
+
+test("createOnlineStream rejects offline-runtime sessions and dead servers", () => {
+ const offline = onlineWsServerAt(1);
+ offline.modelRuntime = "offline";
+ assert.throws(() => offline.createOnlineStream(), /online-runtime/);
+
+ const dead = onlineWsServerAt(1);
+ dead.ready = false;
+ assert.throws(() => dead.createOnlineStream(), /not running/);
+});
+
+test("abort closes the stream without waiting for the server", async () => {
+ const mock = await startMockOnlineServer({});
+ try {
+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});
+ stream.sendPcm16(Buffer.alloc(3200));
+ stream.abort();
+ const { text } = await stream.finish();
+ assert.equal(text, "");
+ } finally {
+ await mock.close();
+ }
+});
+
+test("pcm16ToFloat32 converts int16 samples to normalized float32", () => {
+ const pcm = Buffer.alloc(8);
+ pcm.writeInt16LE(0, 0);
+ pcm.writeInt16LE(16384, 2);
+ pcm.writeInt16LE(-16384, 4);
+ pcm.writeInt16LE(-32768, 6);
+
+ const floats = pcm16ToFloat32(pcm);
+
+ assert.ok(floats instanceof Float32Array);
+ assert.deepEqual(Array.from(floats), [0, 0.5, -0.5, -1]);
+});
diff --git a/test/helpers/parakeetWsResult.test.js b/test/helpers/parakeetWsResult.test.js
new file mode 100644
index 000000000..a583034fd
--- /dev/null
+++ b/test/helpers/parakeetWsResult.test.js
@@ -0,0 +1,56 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const { parseOfflineMessage, parseOnlineMessages } = require("../../src/helpers/parakeetWsResult");
+
+test("offline sherpa message extracts JSON text", () => {
+ assert.equal(parseOfflineMessage(JSON.stringify({ text: "hello world" })), "hello world");
+ assert.equal(parseOfflineMessage("2026"), "2026");
+});
+
+test("online sherpa messages keep finalized segments and latest partial", () => {
+ assert.equal(
+ parseOnlineMessages([
+ JSON.stringify({ text: "hello", is_final: false, segment: 0 }),
+ JSON.stringify({ text: "hello world", is_final: true, segment: 0 }),
+ JSON.stringify({ text: "again", is_final: false, segment: 1 }),
+ ]),
+ "hello world again"
+ );
+});
+
+test("online sherpa parser keeps numeric final text as text", () => {
+ assert.equal(
+ parseOnlineMessages([JSON.stringify({ text: "2026", is_final: true, segment: 0 })]),
+ "2026"
+ );
+});
+
+test("online sherpa parser ignores JSON scalar control messages", () => {
+ assert.equal(
+ parseOnlineMessages([
+ "null",
+ "true",
+ JSON.stringify({ text: "hello", is_final: true, segment: 0 }),
+ ]),
+ "hello"
+ );
+});
+
+test("online sherpa parser keeps latest partial before ignored trailing messages", () => {
+ assert.equal(
+ parseOnlineMessages([
+ JSON.stringify({ text: "hello", is_final: false, segment: 0 }),
+ "null",
+ JSON.stringify({ text: "", is_final: false, segment: 0 }),
+ ]),
+ "hello"
+ );
+});
+
+test("online sherpa parser tolerates non-string text payloads", () => {
+ assert.equal(
+ parseOnlineMessages([JSON.stringify({ text: 2026, is_final: true, segment: 0 })]),
+ "2026"
+ );
+});