Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat(parakeet): add NVIDIA Nemotron streaming ASR models (EN + multilingual 3.5) (#1131)
Gabriel Steine4e5d19ddb35
diff --git a/CLAUDE.md b/CLAUDE.mdindex 9865eb85f..550876cfe 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -179,6 +179,12 @@ OpenWhispr is an Electron-based desktop dictation application that uses whisper.- **Available Models**:- `parakeet-tdt-0.6b-v3`: Multilingual (25 languages), ~680MB- `parakeet-unified-en-0.6b`: English-only, ~631MB, state-of-the-art EN accuracy (5.91% avg WER on Open ASR Leaderboard)+ - `nemotron-speech-streaming-en-0.6b`: English-only, ~632MB, cache-aware streaming FastConformer (`"runtime": "online"` in the registry)+ - `nemotron-3.5-asr-streaming-0.6b`: Multilingual (15 transcription-ready languages, auto detection), ~650MB, cache-aware streaming FastConformer (`"runtime": "online"`)++- **Runtimes**: Models are `offline` (default) or `online` per their registry `runtime` field. Offline models use the bundled `sherpa-onnx-ws-{platform}-{arch}` (offline websocket server); online models use `sherpa-onnx-online-ws-{platform}-{arch}` (online websocket server). Both are downloaded by `scripts/download-sherpa-onnx.js`. The final transcription path still records-then-transcribes; audio is chunked over the websocket and partial/final JSON results are merged by `parakeetWsResult.js`.++- **Live Transcription Preview**: When the preview toggle is on and an online-runtime model is selected, the preview uses a persistent websocket stream (`createOnlineStream` in `parakeetWsServer.js`): worklet PCM is fed as it is captured (`sendPcm16` converts to the float32 wire format inside the ws layer), and partial results update the preview window live (replacing text via `showTranscriptionPreview`). Offline models keep the 1.5s buffered-chunk path (appending via `appendTranscriptionPreview`). If the stream can't start, the preview falls back to the chunked path. Tests: `test/helpers/parakeetOnlineStream.test.js` (mock websocket server).- **Download URLs**: Models from sherpa-onnx ASR models release on GitHubdiff --git a/package.json b/package.jsonindex f8a2f3862..834a84521 100644--- a/package.json+++ b/package.json@@ -20,11 +20,11 @@"compile:mic-listener": "node scripts/build-macos-mic-listener.js","compile:audio-tap": "node scripts/build-macos-audio-tap.js","compile:native": "npm run compile:globe && npm run compile:fast-paste && npm run compile:winkeys && npm run compile:linuxkeys && npm run compile:winpaste && npm run compile:linux-paste && npm run compile:linux-system-audio && npm run compile:text-monitor && npm run compile:media-remote && npm run compile:mic-listener && npm run compile:audio-tap",- "prestart": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",+ "prestart": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model","start": "electron .",- "predev": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",+ "predev": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model","dev": "concurrently -k -r \"npm:dev:renderer\" \"npm:dev:main\"",- "predev:main": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model",+ "predev:main": "npm run compile:native && npm run download:meeting-aec-helper && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model && npm run download:whisper-vad-model","dev:main": "cross-env NODE_ENV=development node scripts/run-electron.js --dev","dev:renderer": "cd src && vite","prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:meeting-aec-helper && npm run download:whisper-vad-model && npm run download:embedding-model -- --for-build && npm run download:diarization-models -- --output-dir resources/bin/diarization-models",diff --git a/scripts/download-sherpa-onnx.js b/scripts/download-sherpa-onnx.jsindex 5d072bd5c..8778cef73 100644--- a/scripts/download-sherpa-onnx.js+++ b/scripts/download-sherpa-onnx.js@@ -2,9 +2,15 @@const fs = require("fs");const path = require("path");const { execFileSync } = require("child_process");-const { downloadFile, findBinaryInDir, parseArgs, setExecutable } = require("./lib/download-utils");--const SHERPA_ONNX_VERSION = "1.12.23";+const {+ cleanupFiles,+ downloadFile,+ findBinaryInDir,+ parseArgs,+ setExecutable,+} = require("./lib/download-utils");++const SHERPA_ONNX_VERSION = "1.13.4";const GITHUB_RELEASE_URL = `https://github.com/k2-fsa/sherpa-onnx/releases/download/v${SHERPA_ONNX_VERSION}`;// Binary configurations for each platform@@ -14,6 +20,8 @@ const BINARIES = {archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-osx-universal2-shared.tar.bz2`,binaryPath: "sherpa-onnx-offline-websocket-server",outputName: "sherpa-onnx-ws-darwin-arm64",+ onlineBinaryPath: "sherpa-onnx-online-websocket-server",+ onlineOutputName: "sherpa-onnx-online-ws-darwin-arm64",diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization",diarizeOutputName: "sherpa-onnx-diarize-darwin-arm64",libPattern: "*.dylib",@@ -22,14 +30,19 @@ const BINARIES = {archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-osx-universal2-shared.tar.bz2`,binaryPath: "sherpa-onnx-offline-websocket-server",outputName: "sherpa-onnx-ws-darwin-x64",+ onlineBinaryPath: "sherpa-onnx-online-websocket-server",+ onlineOutputName: "sherpa-onnx-online-ws-darwin-x64",diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization",diarizeOutputName: "sherpa-onnx-diarize-darwin-x64",libPattern: "*.dylib",},"win32-x64": {- archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-win-x64-shared.tar.bz2`,+ // Since 1.13.4 the Windows assets carry an MSVC runtime/build-type suffix+ archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-win-x64-shared-MD-Release.tar.bz2`,binaryPath: "sherpa-onnx-offline-websocket-server.exe",outputName: "sherpa-onnx-ws-win32-x64.exe",+ onlineBinaryPath: "sherpa-onnx-online-websocket-server.exe",+ onlineOutputName: "sherpa-onnx-online-ws-win32-x64.exe",diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization.exe",diarizeOutputName: "sherpa-onnx-diarize-win32-x64.exe",libPattern: "*.dll",@@ -38,6 +51,8 @@ const BINARIES = {archiveName: `sherpa-onnx-v${SHERPA_ONNX_VERSION}-linux-x64-shared.tar.bz2`,binaryPath: "sherpa-onnx-offline-websocket-server",outputName: "sherpa-onnx-ws-linux-x64",+ onlineBinaryPath: "sherpa-onnx-online-websocket-server",+ onlineOutputName: "sherpa-onnx-online-ws-linux-x64",diarizeBinaryPath: "sherpa-onnx-offline-speaker-diarization",diarizeOutputName: "sherpa-onnx-diarize-linux-x64",libPattern: "*.so*",@@ -46,6 +61,14 @@ const BINARIES = {const BIN_DIR = path.join(__dirname, "..", "resources", "bin");+const VERSIONED_LIB_PATTERN = /^(lib.+?)\.(\d+\.\d+\.\d+)\.(dylib|so|dll)$/;++// Upstream 1.13.4 ships an invalid arm64 signature on libonnxruntime; dyld SIGKILLs unsigned loads.+function adhocSign(filePath, platformArch) {+ if (process.platform !== "darwin" || !platformArch.startsWith("darwin")) return;+ execFileSync("codesign", ["--force", "--sign", "-", filePath], { stdio: "ignore" });+}+function getDownloadUrl(archiveName) {return `${GITHUB_RELEASE_URL}/${archiveName}`;}@@ -95,6 +118,37 @@ function matchesPattern(filename, pattern) {return false;}+function copyBinary(extractDir, binaryName, outputPath, platformArch) {+ const foundPath = findBinaryInDir(extractDir, binaryName);++ if (!foundPath || !fs.existsSync(foundPath)) {+ console.error(` ${platformArch}: Binary '${binaryName}' not found in archive`);+ return false;+ }++ fs.rmSync(outputPath, { force: true });+ fs.copyFileSync(foundPath, outputPath);+ setExecutable(outputPath);+ adhocSign(outputPath, platformArch);+ console.log(` ${platformArch}: Extracted to ${path.basename(outputPath)}`);+ return true;+}++function isCompleteInstall(markerPath, binaryPaths) {+ if (binaryPaths.some((binaryPath) => !fs.existsSync(binaryPath))) return false;++ try {+ const marker = JSON.parse(fs.readFileSync(markerPath, "utf8"));+ return (+ marker.version === SHERPA_ONNX_VERSION &&+ Array.isArray(marker.libraries) &&+ marker.libraries.every((lib) => fs.existsSync(path.join(BIN_DIR, lib)))+ );+ } catch {+ return false;+ }+}+async function downloadBinary(platformArch, config, isForce = false) {if (!config) {console.log(` ${platformArch}: Not supported`);@@ -102,54 +156,38 @@ async function downloadBinary(platformArch, config, isForce = false) {}const outputPath = path.join(BIN_DIR, config.outputName);+ const onlineOutputPath = path.join(BIN_DIR, config.onlineOutputName);const diarizeOutputPath = path.join(BIN_DIR, config.diarizeOutputName);+ const installMarkerPath = path.join(BIN_DIR, `.sherpa-onnx-${platformArch}.json`);- if (fs.existsSync(outputPath) && fs.existsSync(diarizeOutputPath) && !isForce) {+ if (!isForce && isCompleteInstall(installMarkerPath, [outputPath, onlineOutputPath, diarizeOutputPath])) {console.log(` ${platformArch}: Already exists (use --force to re-download)`);return true;}+ if (isForce && fs.existsSync(installMarkerPath)) fs.unlinkSync(installMarkerPath);const url = getDownloadUrl(config.archiveName);console.log(` ${platformArch}: Downloading from ${url}`);const archivePath = path.join(BIN_DIR, config.archiveName);+ const extractDir = path.join(BIN_DIR, `temp-sherpa-${platformArch}`);try {await downloadFile(url, archivePath);- const extractDir = path.join(BIN_DIR, `temp-sherpa-${platformArch}`);fs.mkdirSync(extractDir, { recursive: true });extractTarBz2(archivePath, extractDir);- // Find the WebSocket server binary (may be in a subdirectory)- const binaryName = path.basename(config.binaryPath);- let binaryPath = findBinaryInDir(extractDir, binaryName);-- if (binaryPath && fs.existsSync(binaryPath)) {- fs.copyFileSync(binaryPath, outputPath);- setExecutable(outputPath);- console.log(` ${platformArch}: Extracted to ${config.outputName}`);- } else {- console.error(` ${platformArch}: Binary '${binaryName}' not found in archive`);- return false;- }-- // Find the diarization binary from the same archive- const diarizeBinaryName = path.basename(config.diarizeBinaryPath);- let diarizeBinaryPath = findBinaryInDir(extractDir, diarizeBinaryName);-- if (diarizeBinaryPath && fs.existsSync(diarizeBinaryPath)) {- fs.copyFileSync(diarizeBinaryPath, diarizeOutputPath);- setExecutable(diarizeOutputPath);- console.log(` ${platformArch}: Extracted to ${config.diarizeOutputName}`);- } else {- console.error(- ` ${platformArch}: Diarization binary '${diarizeBinaryName}' not found in archive`- );- return false;+ for (const [binaryName, destPath] of [+ [config.binaryPath, outputPath],+ [config.onlineBinaryPath, onlineOutputPath],+ [config.diarizeBinaryPath, diarizeOutputPath],+ ]) {+ if (!copyBinary(extractDir, binaryName, destPath, platformArch)) return false;}// Copy shared libraries+ const copiedLibraries = [];if (config.libPattern) {const libraries = findLibrariesInDir(extractDir, config.libPattern);@@ -161,15 +199,17 @@ async function downloadBinary(platformArch, config, isForce = false) {const libName = path.basename(libPath);const destPath = path.join(BIN_DIR, libName);- // Detect versioned dylib pattern: libFoo.X.Y.Z.dylib- const versionMatch = libName.match(/^(lib.+?)\.(\d+\.\d+\.\d+)\.(dylib|so|dll)$/);+ const versionMatch = libName.match(VERSIONED_LIB_PATTERN);if (versionMatch) {- const baseName = `${versionMatch[1]}.${versionMatch[3]}`; // e.g. libonnxruntime.dylib- versionedLibs.set(baseName, libName);+ versionedLibs.set(`${versionMatch[1]}.${versionMatch[3]}`, libName);}+ // rm first: copying onto an existing symlink would write through it+ fs.rmSync(destPath, { force: true });fs.copyFileSync(libPath, destPath);setExecutable(destPath);+ adhocSign(destPath, platformArch);+ copiedLibraries.push(libName);console.log(` ${platformArch}: Copied library ${libName}`);}@@ -177,28 +217,32 @@ async function downloadBinary(platformArch, config, isForce = false) {if (process.platform !== "win32") {for (const [baseName, versionedName] of versionedLibs) {const basePath = path.join(BIN_DIR, baseName);- const versionedPath = path.join(BIN_DIR, versionedName);- if (- fs.existsSync(basePath) &&- fs.existsSync(versionedPath) &&- !fs.lstatSync(basePath).isSymbolicLink()- ) {- fs.unlinkSync(basePath);- fs.symlinkSync(versionedName, basePath);- console.log(` ${platformArch}: Symlinked ${baseName} -> ${versionedName}`);+ fs.rmSync(basePath, { force: true });+ fs.symlinkSync(versionedName, basePath);+ console.log(` ${platformArch}: Symlinked ${baseName} -> ${versionedName}`);++ for (const file of fs.readdirSync(BIN_DIR)) {+ const match = file.match(VERSIONED_LIB_PATTERN);+ if (match && `${match[1]}.${match[3]}` === baseName && file !== versionedName) {+ fs.unlinkSync(path.join(BIN_DIR, file));+ console.log(` ${platformArch}: Removed stale ${file}`);+ }}}}}- // Cleanup- fs.rmSync(extractDir, { recursive: true, force: true });- if (fs.existsSync(archivePath)) fs.unlinkSync(archivePath);+ fs.writeFileSync(+ installMarkerPath,+ JSON.stringify({ version: SHERPA_ONNX_VERSION, libraries: copiedLibraries })+ );return true;} catch (error) {console.error(` ${platformArch}: Failed - ${error.message}`);- if (fs.existsSync(archivePath)) fs.unlinkSync(archivePath);return false;+ } finally {+ fs.rmSync(extractDir, { recursive: true, force: true });+ if (fs.existsSync(archivePath)) fs.unlinkSync(archivePath);}}@@ -216,8 +260,9 @@ async function main() {return;}+ const config = BINARIES[args.platformArch];console.log(`Downloading for target platform (${args.platformArch}):`);- const ok = await downloadBinary(args.platformArch, BINARIES[args.platformArch], args.isForce);+ const ok = await downloadBinary(args.platformArch, config, args.isForce);if (!ok) {console.error(`Failed to download binaries for ${args.platformArch}`);process.exitCode = 1;@@ -235,16 +280,11 @@ async function main() {}if (args.shouldCleanup) {- const wsPrefix = `sherpa-onnx-ws-${args.platformArch}`;- const diarizePrefix = `sherpa-onnx-diarize-${args.platformArch}`;- const files = fs.readdirSync(BIN_DIR).filter((f) => f.startsWith("sherpa-onnx"));- files.forEach((file) => {- if (!file.startsWith(wsPrefix) && !file.startsWith(diarizePrefix)) {- const filePath = path.join(BIN_DIR, file);- console.log(`Removing old binary: ${file}`);- fs.unlinkSync(filePath);- }- });+ cleanupFiles(BIN_DIR, "sherpa-onnx", [+ `sherpa-onnx-ws-${args.platformArch}`,+ `sherpa-onnx-online-ws-${args.platformArch}`,+ `sherpa-onnx-diarize-${args.platformArch}`,+ ]);}} else {console.log("Downloading binaries for all platforms:");diff --git a/scripts/lib/download-utils.js b/scripts/lib/download-utils.jsindex 0c40f33b0..a3cc6e9f7 100644--- a/scripts/lib/download-utils.js+++ b/scripts/lib/download-utils.js@@ -320,12 +320,13 @@ function setExecutable(filePath) {}function cleanupFiles(binDir, prefix, keepPrefix) {+ const keepPrefixes = Array.isArray(keepPrefix) ? keepPrefix : [keepPrefix];// Never delete shared libraries; only platform binaries. The b9763 split ships// llama-server-impl.dll, which shares the "llama-server" prefix and must survive.const isLibrary = (f) => /\.(dll|dylib)$/i.test(f) || /\.so(\.\d+)*$/.test(f);const files = fs.readdirSync(binDir).filter((f) => f.startsWith(prefix));files.forEach((file) => {- if (!file.startsWith(keepPrefix) && !isLibrary(file)) {+ if (!keepPrefixes.some((keep) => file.startsWith(keep)) && !isLibrary(file)) {const filePath = path.join(binDir, file);console.log(`Removing old binary: ${file}`);fs.unlinkSync(filePath);diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 6150be7d1..cf0fb1d7b 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -5508,12 +5508,21 @@ class IPCHandlers {let dictationPreviewLanguage = null;let dictationPreviewSessionActive = false;let dictationPreviewChunkCount = 0;+ // Online-runtime models stream here instead of the 1.5s chunked path.+ let dictationPreviewStream = null;+ // Bumped on every reset so async preview work can detect a stale session.+ let dictationPreviewGen = 0;const resetDictationPreviewState = ({ preserveSession = false } = {}) => {+ dictationPreviewGen++;if (dictationPreviewTimer) {clearInterval(dictationPreviewTimer);dictationPreviewTimer = null;}+ if (dictationPreviewStream) {+ dictationPreviewStream.abort();+ dictationPreviewStream = null;+ }dictationPreviewMode = false;if (!preserveSession) {dictationPreviewSessionActive = false;@@ -6208,6 +6217,7 @@ class IPCHandlers {ipcMain.handle("start-dictation-preview", async (_event, { provider, model, language }) => {resetDictationPreviewState();+ const gen = dictationPreviewGen;dictationPreviewMode = true;dictationPreviewSessionActive = true;dictationPreviewProvider = provider;@@ -6215,6 +6225,35 @@ class IPCHandlers {dictationPreviewLanguage = language || null;dictationPreviewChunkCount = 0;this.windowManager.showTranscriptionPreview("");++ if (provider === "nvidia" && this.parakeetManager.supportsOnlineStreaming(model)) {+ try {+ const stream = await this.parakeetManager.createOnlineStream(model, {+ onUpdate: (text) => {+ if (gen === dictationPreviewGen && text) {+ this.windowManager.showTranscriptionPreview(text);+ }+ },+ });+ if (gen !== dictationPreviewGen) {+ stream.abort();+ return { success: true };+ }+ dictationPreviewStream = stream;+ for (const chunk of dictationPreviewBuffer) {+ stream.sendPcm16(chunk);+ }+ dictationPreviewBuffer = [];+ return { success: true };+ } catch (error) {+ debugLogger.warn("Online preview stream unavailable, falling back to chunked preview", {+ model,+ error: error.message,+ });+ }+ }++ if (gen !== dictationPreviewGen) return { success: true };dictationPreviewTimer = setInterval(() => transcribeDictationPreviewChunk(), 1500);return { success: true };});@@ -6229,9 +6268,12 @@ class IPCHandlers {bufferSize: dictationPreviewBuffer.length,});}- dictationPreviewBuffer.push(- Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer)- );+ const pcm = Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer);+ if (dictationPreviewStream) {+ dictationPreviewStream.sendPcm16(pcm);+ return;+ }+ dictationPreviewBuffer.push(pcm);});ipcMain.handle("dismiss-dictation-preview", async () => {@@ -6272,7 +6314,16 @@ class IPCHandlers {}clearInterval(dictationPreviewTimer);dictationPreviewTimer = null;- await transcribeDictationPreviewChunk();+ if (dictationPreviewStream) {+ const stream = dictationPreviewStream;+ dictationPreviewStream = null;+ const { text } = await stream.finish().catch(() => ({ text: "" }));+ if (text && dictationPreviewSessionActive) {+ this.windowManager.showTranscriptionPreview(text);+ }+ } else {+ await transcribeDictationPreviewChunk();+ }resetDictationPreviewState({ preserveSession: true });if (!dictationPreviewSessionActive) {return { success: true };diff --git a/src/helpers/liveSpeakerIdentifier.js b/src/helpers/liveSpeakerIdentifier.jsindex 5059c1d8d..d39dd620c 100644--- a/src/helpers/liveSpeakerIdentifier.js+++ b/src/helpers/liveSpeakerIdentifier.js@@ -2,7 +2,7 @@ const fs = require("fs");const debugLogger = require("./debugLogger");const speakerEmbeddings = require("./speakerEmbeddings");const { MAX_EMBEDDING_SECONDS } = speakerEmbeddings;-const { downsample24kTo16k } = require("../utils/audioUtils");+const { downsample24kTo16k, pcm16ToFloat32 } = require("../utils/audioUtils");const { MAX_SPEAKER_COUNT } = require("../constants/speakerDetection.json");function clampMaxSpeakers(value) {@@ -31,17 +31,6 @@ const MATCH_MARGIN = 0.03;const LIVE_WINDOW_PADDING_SECONDS = 0.75;const DEFAULT_VAD_STATE_SHAPE = [2, 1, 64];-function pcm16BufferToFloat32(buffer) {- const input = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);- const output = new Float32Array(input.length);-- for (let i = 0; i < input.length; i += 1) {- output[i] = input[i] / 32768;- }-- return output;-}-function appendFloat32(existing, next) {if (!existing.length) return next;if (!next.length) return existing;@@ -424,7 +413,7 @@ class LiveSpeakerIdentifier {);if (!downsampled.length) return;- this.audioRemainder = appendFloat32(this.audioRemainder, pcm16BufferToFloat32(downsampled));+ this.audioRemainder = appendFloat32(this.audioRemainder, pcm16ToFloat32(downsampled));while (this.audioRemainder.length >= VAD_WINDOW_SIZE) {const window = this.audioRemainder.subarray(0, VAD_WINDOW_SIZE);diff --git a/src/helpers/meetingEchoLeakDetector.js b/src/helpers/meetingEchoLeakDetector.jsindex 35500e09b..91b9c4459 100644--- a/src/helpers/meetingEchoLeakDetector.js+++ b/src/helpers/meetingEchoLeakDetector.js@@ -30,16 +30,7 @@ const DOUBLE_TALK_EXPLAINED = 0.3;const DOUBLE_TALK_RESIDUAL = 0.2;const DOUBLE_TALK_MIC_TO_SYSTEM_RATIO = 1.16;-function pcm16BufferToFloat32(buffer) {- const input = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);- const output = new Float32Array(input.length);-- for (let i = 0; i < input.length; i += 1) {- output[i] = input[i] / 32768;- }-- return output;-}+const { pcm16ToFloat32 } = require("../utils/audioUtils");function computeRms(samples) {if (!samples.length) {@@ -70,7 +61,7 @@ class MeetingEchoLeakDetector {return;}- const samples = pcm16BufferToFloat32(buffer);+ const samples = pcm16ToFloat32(buffer);if (!samples.length) {return;}@@ -90,7 +81,7 @@ class MeetingEchoLeakDetector {return null;}- const samples = pcm16BufferToFloat32(buffer);+ const samples = pcm16ToFloat32(buffer);if (!samples.length) {return null;}diff --git a/src/helpers/parakeet.js b/src/helpers/parakeet.jsindex e01a0c59b..3fe0f18f5 100644--- a/src/helpers/parakeet.js+++ b/src/helpers/parakeet.js@@ -14,6 +14,7 @@ const ParakeetServerManager = require("./parakeetServer");const { getModelsDirForService } = require("./modelDirUtils");const modelRegistryData = require("../models/modelRegistryData.json");+const { getModelRuntime, REQUIRED_MODEL_FILES } = require("./parakeetModelInfo");function getParakeetModelConfig(modelName) {const modelInfo = modelRegistryData.parakeetModels[modelName];@@ -72,7 +73,7 @@ class ParakeetManager {if (localTranscriptionProvider === "nvidia" &¶keetModel &&- this.serverManager.isAvailable()+ this.serverManager.isAvailable(getModelRuntime(parakeetModel))) {if (this.serverManager.isModelDownloaded(parakeetModel)) {debugLogger.info("Pre-warming parakeet server", { model: parakeetModel });@@ -112,15 +113,16 @@ class ParakeetManager {debugLogger.info("Parakeet initialization complete", {totalTimeMs: Date.now() - startTime,- binaryAvailable: this.serverManager.isAvailable(),+ binaryAvailable: this.serverManager.hasAnyWsBinary(),});}async logDependencyStatus() {const status = {sherpaOnnx: {- available: this.serverManager.isAvailable(),- path: this.serverManager.getBinaryPath(),+ available: this.serverManager.hasAnyWsBinary(),+ path:+ this.serverManager.getBinaryPath("offline") || this.serverManager.getBinaryPath("online"),},models: [],};@@ -154,16 +156,13 @@ class ParakeetManager {}async checkInstallation() {- const binaryPath = this.serverManager.getBinaryPath();+ const binaryPath =+ this.serverManager.getBinaryPath("offline") || this.serverManager.getBinaryPath("online");if (!binaryPath) {return { installed: false, working: false };}- return {- installed: true,- working: this.serverManager.isAvailable(),- path: binaryPath,- };+ return { installed: true, working: true, path: binaryPath };}async startServer(modelName) {@@ -179,22 +178,36 @@ class ParakeetManager {return this.serverManager.getServerStatus();}+ supportsOnlineStreaming(modelName) {+ return getModelRuntime(modelName) === "online";+ }++ async createOnlineStream(modelName, options = {}) {+ this.validateModelName(modelName);+ const started = await this.serverManager.startServer(modelName);+ if (!started.success) {+ throw new Error(started.reason || "Failed to start parakeet streaming server");+ }+ return this.serverManager.createOnlineStream(options);+ }+async transcribeLocalParakeet(audioBlob, options = {}) {+ const model = options.model || "parakeet-tdt-0.6b-v3";+ const serverAvailable = this.serverManager.isAvailable(getModelRuntime(model));+debugLogger.logSTTPipeline("transcribeLocalParakeet - start", {options,audioBlobType: audioBlob?.constructor?.name,audioBlobSize: audioBlob?.byteLength || audioBlob?.size || 0,- serverAvailable: this.serverManager.isAvailable(),+ serverAvailable,});- if (!this.serverManager.isAvailable()) {+ if (!serverAvailable) {throw new Error("sherpa-onnx binary not found. Please ensure the app is installed correctly.");}- const model = options.model || "parakeet-tdt-0.6b-v3";-if (!this.serverManager.isModelDownloaded(model)) {throw new Error(`Parakeet model "${model}" not downloaded. Please download it from Settings.`@@ -341,7 +354,7 @@ class ParakeetManager {progressCallback({ type: "complete", model: modelName, percentage: 100 });}- if (this.serverManager.isAvailable()) {+ if (this.serverManager.isAvailable(getModelRuntime(modelName))) {this.serverManager.startServer(modelName).catch((err) => {debugLogger.warn("Post-download server pre-warm failed (non-fatal)", {error: err.message,@@ -392,7 +405,10 @@ class ParakeetManager {for (const entry of entries) {const entryPath = path.join(extractDir, entry);const stat = await fsPromises.stat(entryPath);- if (stat.isDirectory() && entry.includes("parakeet")) {+ if (+ stat.isDirectory() &&+ REQUIRED_MODEL_FILES.every((file) => fs.existsSync(path.join(entryPath, file)))+ ) {modelDir = entry;break;}@@ -411,13 +427,7 @@ class ParakeetManager {}}- const requiredFiles = [- "encoder.int8.onnx",- "decoder.int8.onnx",- "joiner.int8.onnx",- "tokens.txt",- ];- const missing = requiredFiles.filter((f) => !fs.existsSync(path.join(targetDir, f)));+ const missing = REQUIRED_MODEL_FILES.filter((f) => !fs.existsSync(path.join(targetDir, f)));if (missing.length > 0) {throw new Error(`Extracted model is missing required files: ${missing.join(", ")}`);}@@ -600,8 +610,8 @@ class ParakeetManager {modelsDir: this.getModelsDir(),models: [],};-- const binaryPath = this.serverManager.getBinaryPath();+ const binaryPath =+ this.serverManager.getBinaryPath("offline") || this.serverManager.getBinaryPath("online");if (binaryPath) {diagnostics.sherpaOnnx = { available: true, path: binaryPath };}diff --git a/src/helpers/parakeetModelInfo.js b/src/helpers/parakeetModelInfo.jsnew file mode 100644index 000000000..abea60fd5--- /dev/null+++ b/src/helpers/parakeetModelInfo.js@@ -0,0 +1,14 @@+const modelRegistryData = require("../models/modelRegistryData.json");++const REQUIRED_MODEL_FILES = [+ "encoder.int8.onnx",+ "decoder.int8.onnx",+ "joiner.int8.onnx",+ "tokens.txt",+];++function getModelRuntime(modelName) {+ return modelRegistryData.parakeetModels?.[modelName]?.runtime === "online" ? "online" : "offline";+}++module.exports = { REQUIRED_MODEL_FILES, getModelRuntime };diff --git a/src/helpers/parakeetServer.js b/src/helpers/parakeetServer.jsindex b390f5e4d..4d3d8be16 100644--- a/src/helpers/parakeetServer.js+++ b/src/helpers/parakeetServer.js@@ -11,6 +11,7 @@ const {} = require("./ffmpegUtils");const { getSafeTempDir } = require("./safeTempDir");const ParakeetWsServer = require("./parakeetWsServer");+const { getModelRuntime, REQUIRED_MODEL_FILES } = require("./parakeetModelInfo");const SAMPLE_RATE = 16000;const BYTES_PER_SAMPLE = 4; // float32@@ -23,12 +24,16 @@ class ParakeetServerManager {this.wsServer = new ParakeetWsServer();}- getBinaryPath() {- return this.wsServer.getWsBinaryPath();+ getBinaryPath(runtime) {+ return this.wsServer.getWsBinaryPath(runtime);}- isAvailable() {- return this.wsServer.isAvailable();+ isAvailable(runtime) {+ return this.wsServer.isAvailable(runtime);+ }++ hasAnyWsBinary() {+ return this.wsServer.hasAnyWsBinary();}getModelsDir() {@@ -37,22 +42,9 @@ class ParakeetServerManager {isModelDownloaded(modelName) {const modelDir = path.join(this.getModelsDir(), modelName);- const requiredFiles = [- "encoder.int8.onnx",- "decoder.int8.onnx",- "joiner.int8.onnx",- "tokens.txt",- ];-if (!fs.existsSync(modelDir)) return false;- for (const file of requiredFiles) {- if (!fs.existsSync(path.join(modelDir, file))) {- return false;- }- }-- return true;+ return REQUIRED_MODEL_FILES.every((file) => fs.existsSync(path.join(modelDir, file)));}async _ensureWav(audioBuffer) {@@ -99,7 +91,7 @@ class ParakeetServerManager {const { wavBuffer, filesToCleanup } = await this._ensureWav(audioBuffer);try {if (!this.wsServer.ready || this.wsServer.modelName !== modelName) {- await this.wsServer.start(modelName, modelDir);+ await this.wsServer.start(modelName, modelDir, getModelRuntime(modelName));}const samples = wavToFloat32Samples(wavBuffer);@@ -168,7 +160,8 @@ class ParakeetServerManager {}async startServer(modelName) {- if (!this.wsServer.isAvailable()) {+ const runtime = getModelRuntime(modelName);+ if (!this.wsServer.isAvailable(runtime)) {return { success: false, reason: "parakeet WS server binary not found" };}@@ -178,7 +171,7 @@ class ParakeetServerManager {}try {- await this.wsServer.start(modelName, modelDir);+ await this.wsServer.start(modelName, modelDir, runtime);return { success: true, port: this.wsServer.port };} catch (error) {debugLogger.error("Failed to start parakeet WS server", { error: error.message });@@ -194,12 +187,8 @@ class ParakeetServerManager {return this.wsServer.getStatus();}- getStatus() {- return {- available: this.isAvailable(),- binaryPath: this.getBinaryPath(),- modelsDir: this.getModelsDir(),- };+ createOnlineStream(options) {+ return this.wsServer.createOnlineStream(options);}}diff --git a/src/helpers/parakeetWsResult.js b/src/helpers/parakeetWsResult.jsnew file mode 100644index 000000000..a916ac812--- /dev/null+++ b/src/helpers/parakeetWsResult.js@@ -0,0 +1,59 @@+function parseOfflineMessage(message) {+ const text = String(message || "").trim();+ try {+ const parsed = JSON.parse(text);+ return typeof parsed?.text === "string" ? parsed.text.trim() : text;+ } catch {+ return text;+ }+}++// Dedupes finalized segments by id; text() is finalized text plus the trailing partial.+function createOnlineAccumulator() {+ const finalizedSegments = new Set();+ let finalizedText = "";+ let partialText = "";+ let fallbackKey = 0;++ const text = () =>+ finalizedText && partialText ? `${finalizedText} ${partialText}` : finalizedText || partialText;++ return {+ push(message) {+ let parsed;+ try {+ parsed = JSON.parse(message);+ } catch {+ parsed = { text: message };+ }+ if (!parsed || typeof parsed !== "object") return text();++ const messageText = String(parsed.text ?? "").trim();+ if (!messageText) return text();++ if (!parsed.is_final) {+ partialText = finalizedSegments.has(parsed.segment) ? "" : messageText;+ return text();+ }++ const segment = parsed.segment ?? `fallback:${fallbackKey++}`;+ if (!finalizedSegments.has(segment)) {+ finalizedSegments.add(segment);+ finalizedText = finalizedText ? `${finalizedText} ${messageText}` : messageText;+ }+ partialText = "";+ return text();+ },+ text,+ };+}++function parseOnlineMessages(messages) {+ const accumulator = createOnlineAccumulator();+ for (const message of messages) {+ accumulator.push(message);+ }+ return accumulator.text();+}++module.exports = { parseOfflineMessage, parseOnlineMessages, createOnlineAccumulator };diff --git a/src/helpers/parakeetWsServer.js b/src/helpers/parakeetWsServer.jsindex b6366ace4..2a0b8d243 100644--- a/src/helpers/parakeetWsServer.js+++ b/src/helpers/parakeetWsServer.js@@ -11,12 +11,16 @@ const {} = require("../utils/serverUtils");const { getSafeTempDir } = require("./safeTempDir");const sidecarPidFile = require("./sidecarPidFile");+const { parseOfflineMessage, createOnlineAccumulator } = require("./parakeetWsResult");+const { pcm16ToFloat32 } = require("../utils/audioUtils");const PORT_RANGE_START = 6006;const PORT_RANGE_END = 6029;const STARTUP_TIMEOUT_MS = 60000;const HEALTH_CHECK_INTERVAL_MS = 5000;const TRANSCRIPTION_TIMEOUT_MS = 300000;+const ONLINE_CHUNK_BYTES = 8000 * 4;+const ONLINE_FINISH_TIMEOUT_MS = 10000;class ParakeetWsServer {constructor() {@@ -25,35 +29,39 @@ class ParakeetWsServer {this.ready = false;this.modelName = null;this.modelDir = null;+ this.modelRuntime = "offline";this.startupPromise = null;this.healthCheckInterval = null;- this.cachedWsBinaryPath = null;+ this.cachedBinaryPaths = {};}- getWsBinaryPath() {- if (this.cachedWsBinaryPath) return this.cachedWsBinaryPath;+ getWsBinaryPath(runtime = "offline") {+ if (this.cachedBinaryPaths[runtime]) return this.cachedBinaryPaths[runtime];const platformArch = `${process.platform}-${process.arch}`;+ const prefix = runtime === "online" ? "sherpa-onnx-online-ws" : "sherpa-onnx-ws";const binaryName =- process.platform === "win32"- ? `sherpa-onnx-ws-${platformArch}.exe`- : `sherpa-onnx-ws-${platformArch}`;+ process.platform === "win32" ? `${prefix}-${platformArch}.exe` : `${prefix}-${platformArch}`;const resolved = resolveBinaryPath(binaryName);- if (resolved) this.cachedWsBinaryPath = resolved;+ if (resolved) this.cachedBinaryPaths[runtime] = resolved;return resolved;}- isAvailable() {- return this.getWsBinaryPath() !== null;+ isAvailable(runtime = "offline") {+ return this.getWsBinaryPath(runtime) !== null;}- async start(modelName, modelDir) {+ hasAnyWsBinary() {+ return this.isAvailable("offline") || this.isAvailable("online");+ }++ async start(modelName, modelDir, runtime = "offline") {if (this.startupPromise) return this.startupPromise;if (this.ready && this.modelName === modelName) return;if (this.process) await this.stop();- this.startupPromise = this._doStart(modelName, modelDir);+ this.startupPromise = this._doStart(modelName, modelDir, runtime);try {await this.startupPromise;} finally {@@ -61,25 +69,29 @@ class ParakeetWsServer {}}- async _doStart(modelName, modelDir) {- const wsBinary = this.getWsBinaryPath();- if (!wsBinary) throw new Error("sherpa-onnx WS server binary not found");+ async _doStart(modelName, modelDir, runtime) {+ const wsBinary = this.getWsBinaryPath(runtime);+ if (!wsBinary) throw new Error(`sherpa-onnx ${runtime} WS server binary not found`);if (!fs.existsSync(modelDir)) throw new Error(`Model directory not found: ${modelDir}`);this.port = await findAvailablePort(PORT_RANGE_START, PORT_RANGE_END);this.modelName = modelName;this.modelDir = modelDir;+ this.modelRuntime = runtime;+ const threads = Math.max(1, Math.min(4, Math.floor(os.cpus().length * 0.75)));const args = [`--tokens=${path.join(modelDir, "tokens.txt")}`,`--encoder=${path.join(modelDir, "encoder.int8.onnx")}`,`--decoder=${path.join(modelDir, "decoder.int8.onnx")}`,`--joiner=${path.join(modelDir, "joiner.int8.onnx")}`,`--port=${this.port}`,- `--num-threads=${Math.max(1, Math.min(4, Math.floor(os.cpus().length * 0.75)))}`,+ ...(runtime === "online"+ ? [`--num-work-threads=${threads}`, "--warm-up=0"]+ : [`--num-threads=${threads}`]),];- debugLogger.debug("Starting parakeet WS server", { port: this.port, modelName, args });+ debugLogger.debug("Starting parakeet WS server", { port: this.port, modelName, runtime, args });this.process = spawn(wsBinary, args, {stdio: ["ignore", "pipe", "pipe"],@@ -130,6 +142,7 @@ class ParakeetWsServer {debugLogger.info("parakeet-ws server started successfully", {port: this.port,model: modelName,+ runtime,});await this._warmUp();@@ -210,6 +223,14 @@ class ParakeetWsServer {throw new Error("parakeet-ws server is not running");}+ if (this.modelRuntime === "online") {+ return this._transcribeOnline(samplesBuffer);+ }++ return this._transcribeOffline(samplesBuffer, sampleRate);+ }++ _transcribeOffline(samplesBuffer, sampleRate) {return new Promise((resolve, reject) => {const startTime = Date.now();let result = "";@@ -259,12 +280,7 @@ class ParakeetWsServer {resultPreview: result.slice(0, 200),});- try {- const parsed = JSON.parse(result);- resolve({ text: (parsed.text || "").trim(), elapsed });- } catch {- resolve({ text: result.trim(), elapsed });- }+ resolve({ text: parseOfflineMessage(result), elapsed });});ws.on("error", (error) => {@@ -274,6 +290,142 @@ class ParakeetWsServer {});}+ // samplesBuffer must already be 16kHz float32.+ async _transcribeOnline(samplesBuffer) {+ const startTime = Date.now();+ let streamError = null;+ let timedOut = false;++ const stream = this.createOnlineStream({+ onError: (error) => {+ streamError = error;+ },+ });++ debugLogger.debug("parakeet-ws sending streaming audio", {+ samplesBytes: samplesBuffer.length,+ });+ for (let offset = 0; offset < samplesBuffer.length; offset += ONLINE_CHUNK_BYTES) {+ stream.sendFloat32(samplesBuffer.subarray(offset, offset + ONLINE_CHUNK_BYTES));+ }++ const timeout = setTimeout(() => {+ timedOut = true;+ stream.abort();+ }, TRANSCRIPTION_TIMEOUT_MS);+ try {+ const { text } = await stream.finish();+ if (timedOut) throw new Error("parakeet-ws transcription timed out");+ if (streamError) {+ throw new Error(`parakeet-ws transcription failed: ${streamError.message}`);+ }++ const elapsed = Date.now() - startTime;+ debugLogger.debug("parakeet-ws streaming transcription completed", {+ elapsed,+ resultLength: text.length,+ resultPreview: text.slice(0, 200),+ });+ return { text, elapsed };+ } finally {+ clearTimeout(timeout);+ }+ }++ createOnlineStream({ onUpdate, onError } = {}) {+ if (!this.ready || !this.process) {+ throw new Error("parakeet-ws server is not running");+ }+ if (this.modelRuntime !== "online") {+ throw new Error("createOnlineStream requires an online-runtime model");+ }++ const results = createOnlineAccumulator();+ const pendingChunks = [];+ let finishResolve = null;+ let closed = false;+ let lastEmitted = "";++ const ws = new WebSocket(`ws://127.0.0.1:${this.port}`);++ const settle = () => {+ if (closed) return;+ closed = true;+ if (finishResolve) finishResolve({ text: results.text() });+ };++ const sendFloat32 = (float32Samples) => {+ if (closed || finishResolve) return;+ if (ws.readyState === WebSocket.OPEN) {+ ws.send(float32Samples, (err) => {+ if (err) {+ debugLogger.error("parakeet-ws online stream send error", { error: err.message });+ }+ });+ } else {+ pendingChunks.push(float32Samples);+ }+ };++ ws.on("open", () => {+ for (const chunk of pendingChunks) {+ ws.send(chunk);+ }+ pendingChunks.length = 0;+ if (finishResolve) ws.send("Done");+ });++ ws.on("message", (data) => {+ const message = data.toString();+ if (message === "Done!") {+ ws.close();+ return;+ }+ const text = results.push(message);+ if (!closed && text && text !== lastEmitted) {+ lastEmitted = text;+ onUpdate?.(text);+ }+ });++ ws.on("close", () => settle());++ ws.on("error", (error) => {+ debugLogger.warn("parakeet-ws online stream error", { error: error.message });+ onError?.(error);+ settle();+ });++ return {+ sendFloat32,+ sendPcm16: (pcmBuffer) => sendFloat32(pcm16ToFloat32(pcmBuffer)),+ finish: () =>+ new Promise((resolve) => {+ if (closed) {+ resolve({ text: results.text() });+ return;+ }+ finishResolve = resolve;+ if (ws.readyState === WebSocket.OPEN) ws.send("Done");+ // Backstop in case the server never acknowledges with "Done!".+ setTimeout(() => {+ if (!closed) {+ try {+ ws.close();+ } catch {}+ settle();+ }+ }, ONLINE_FINISH_TIMEOUT_MS).unref?.();+ }),+ abort: () => {+ try {+ ws.close();+ } catch {}+ settle();+ },+ };+ }+async stop() {this.stopHealthCheck();@@ -295,11 +447,12 @@ class ParakeetWsServer {this.port = null;this.modelName = null;this.modelDir = null;+ this.modelRuntime = "offline";}getStatus() {return {- available: this.isAvailable(),+ available: this.hasAnyWsBinary(),running: this.ready && this.process !== null,port: this.port,modelName: this.modelName,diff --git a/src/helpers/sidecarReaper.js b/src/helpers/sidecarReaper.jsindex 6e9633b38..7110016bd 100644--- a/src/helpers/sidecarReaper.js+++ b/src/helpers/sidecarReaper.js@@ -3,11 +3,11 @@ const debugLogger = require("./debugLogger");const sidecarPidFile = require("./sidecarPidFile");const EXPECTED_BINARY_FRAGMENTS = {- parakeet: "sherpa-onnx-ws",- whisper: "whisper-server",- llama: "llama-server",- qdrant: "qdrant",- diarization: "sherpa-onnx-diarize",+ parakeet: ["sherpa-onnx-ws-", "sherpa-onnx-online-ws-"],+ whisper: ["whisper-server"],+ llama: ["llama-server"],+ qdrant: ["qdrant"],+ diarization: ["sherpa-onnx-diarize"],};function isProcessAlive(pid) {@@ -42,8 +42,13 @@ function processCommand(pid) {function reapStaleSidecars() {const entries = sidecarPidFile.readAll();for (const { name, pid } of entries) {- const fragment = EXPECTED_BINARY_FRAGMENTS[name];- if (fragment && isProcessAlive(pid) && processCommand(pid).includes(fragment)) {+ const fragments = EXPECTED_BINARY_FRAGMENTS[name];+ if (fragments && isProcessAlive(pid)) {+ const command = processCommand(pid);+ if (!fragments.some((fragment) => command.includes(fragment))) {+ sidecarPidFile.clear(name);+ continue;+ }debugLogger.warn("Reaping stale sidecar", { name, pid });try {process.kill(pid, "SIGTERM");diff --git a/src/locales/de/translation.json b/src/locales/de/translation.jsonindex f40db87fc..100c2994e 100644--- a/src/locales/de/translation.json+++ b/src/locales/de/translation.json@@ -1878,7 +1878,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "Schnelle mehrsprachige ASR mit automatischer Spracherkennung (25 Sprachen)",- "parakeet_unified_en_0_6b": "Englisch-only ASR mit Spitzen-Genauigkeit (5,91% durchschnittliche WER)"+ "parakeet_unified_en_0_6b": "Englisch-only ASR mit Spitzen-Genauigkeit (5,91% durchschnittliche WER)",+ "nemotron_speech_streaming_en_0_6b": "Englische Streaming-ASR von NVIDIA Nemotron",+ "nemotron_3_5_asr_streaming_0_6b": "Mehrsprachige Streaming-ASR von NVIDIA Nemotron mit automatischer Spracherkennung (15 Sprachen)"},"whisper": {"tiny": "Am schnellsten, geringere Qualität",diff --git a/src/locales/en/translation.json b/src/locales/en/translation.jsonindex 5cee307f0..63b424acb 100644--- a/src/locales/en/translation.json+++ b/src/locales/en/translation.json@@ -1966,7 +1966,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "Fast multilingual ASR with auto language detection (25 languages)",- "parakeet_unified_en_0_6b": "English-only ASR with state-of-the-art accuracy (5.91% avg WER)"+ "parakeet_unified_en_0_6b": "English-only ASR with state-of-the-art accuracy (5.91% avg WER)",+ "nemotron_speech_streaming_en_0_6b": "English streaming ASR from NVIDIA Nemotron",+ "nemotron_3_5_asr_streaming_0_6b": "Multilingual streaming ASR from NVIDIA Nemotron with auto language detection (15 languages)"},"diarization": {"segmentation": "Speech boundary detection for speaker diarization",diff --git a/src/locales/es/translation.json b/src/locales/es/translation.jsonindex 8448c327c..a29a39f96 100644--- a/src/locales/es/translation.json+++ b/src/locales/es/translation.json@@ -1926,7 +1926,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "ASR multilingüe rápido con detección automática de idioma (25 idiomas)",- "parakeet_unified_en_0_6b": "ASR solo en inglés con precisión de vanguardia (5,91% WER promedio)"+ "parakeet_unified_en_0_6b": "ASR solo en inglés con precisión de vanguardia (5,91% WER promedio)",+ "nemotron_speech_streaming_en_0_6b": "ASR en inglés por streaming de NVIDIA Nemotron",+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilingüe por streaming de NVIDIA Nemotron con detección automática de idioma (15 idiomas)"},"whisper": {"tiny": "El más rápido, menor calidad",diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.jsonindex dfbcbea23..e7238fe82 100644--- a/src/locales/fr/translation.json+++ b/src/locales/fr/translation.json@@ -1926,7 +1926,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "ASR multilingue rapide avec détection automatique de langue (25 langues)",- "parakeet_unified_en_0_6b": "ASR anglais uniquement avec précision de pointe (5,91 % WER moyen)"+ "parakeet_unified_en_0_6b": "ASR anglais uniquement avec précision de pointe (5,91 % WER moyen)",+ "nemotron_speech_streaming_en_0_6b": "ASR anglais en streaming de NVIDIA Nemotron",+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilingue en streaming de NVIDIA Nemotron avec détection automatique de langue (15 langues)"},"whisper": {"tiny": "Le plus rapide, qualité plus faible",diff --git a/src/locales/it/translation.json b/src/locales/it/translation.jsonindex 2683b2dc2..382253a1e 100644--- a/src/locales/it/translation.json+++ b/src/locales/it/translation.json@@ -1878,7 +1878,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "ASR multilingue veloce con rilevamento automatico della lingua (25 lingue)",- "parakeet_unified_en_0_6b": "ASR solo inglese con precisione all'avanguardia (5,91% WER medio)"+ "parakeet_unified_en_0_6b": "ASR solo inglese con precisione all'avanguardia (5,91% WER medio)",+ "nemotron_speech_streaming_en_0_6b": "ASR streaming in inglese di NVIDIA Nemotron",+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilingue in streaming di NVIDIA Nemotron con rilevamento automatico della lingua (15 lingue)"},"whisper": {"tiny": "Il più veloce, qualità inferiore",diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.jsonindex ae90fe29d..c21cac917 100644--- a/src/locales/ja/translation.json+++ b/src/locales/ja/translation.json@@ -1878,7 +1878,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "高速多言語 ASR、自動言語検出対応(25 言語)",- "parakeet_unified_en_0_6b": "英語専用の最先端精度ASR(平均WER 5.91%)"+ "parakeet_unified_en_0_6b": "英語専用の最先端精度ASR(平均WER 5.91%)",+ "nemotron_speech_streaming_en_0_6b": "NVIDIA Nemotron の英語ストリーミング ASR",+ "nemotron_3_5_asr_streaming_0_6b": "NVIDIA Nemotron の多言語ストリーミング ASR、自動言語検出対応(15 言語)"},"whisper": {"tiny": "最速、品質は低め",diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.jsonindex 1a64da157..7b66b6b59 100644--- a/src/locales/pt/translation.json+++ b/src/locales/pt/translation.json@@ -1857,7 +1857,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "ASR multilíngue rápido com detecção automática de idioma (25 idiomas)",- "parakeet_unified_en_0_6b": "ASR apenas em inglês com precisão de ponta (5,91% WER médio)"+ "parakeet_unified_en_0_6b": "ASR apenas em inglês com precisão de ponta (5,91% WER médio)",+ "nemotron_speech_streaming_en_0_6b": "ASR em inglês por streaming da NVIDIA Nemotron",+ "nemotron_3_5_asr_streaming_0_6b": "ASR multilíngue por streaming da NVIDIA Nemotron com detecção automática de idioma (15 idiomas)"},"whisper": {"tiny": "Mais rápido, qualidade inferior",diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.jsonindex a5fa12f14..62d7d09f4 100644--- a/src/locales/ru/translation.json+++ b/src/locales/ru/translation.json@@ -1878,7 +1878,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "Быстрое многоязычное распознавание с автоопределением языка (25 языков)",- "parakeet_unified_en_0_6b": "ASR только для английского с передовой точностью (средний WER 5,91%)"+ "parakeet_unified_en_0_6b": "ASR только для английского с передовой точностью (средний WER 5,91%)",+ "nemotron_speech_streaming_en_0_6b": "Потоковое распознавание английской речи от NVIDIA Nemotron",+ "nemotron_3_5_asr_streaming_0_6b": "Многоязычное потоковое распознавание речи от NVIDIA Nemotron с автоопределением языка (15 языков)"},"whisper": {"tiny": "Самый быстрый, низкое качество",diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.jsonindex 09393e2d5..94b2dfb2f 100644--- a/src/locales/zh-CN/translation.json+++ b/src/locales/zh-CN/translation.json@@ -1873,7 +1873,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "快速多语言语音识别,自动语言检测(25 种语言)",- "parakeet_unified_en_0_6b": "仅英语 ASR,最先进的准确度(平均 WER 5.91%)"+ "parakeet_unified_en_0_6b": "仅英语 ASR,最先进的准确度(平均 WER 5.91%)",+ "nemotron_speech_streaming_en_0_6b": "NVIDIA Nemotron 英语流式语音识别",+ "nemotron_3_5_asr_streaming_0_6b": "NVIDIA Nemotron 多语言流式语音识别,自动语言检测(15 种语言)"},"whisper": {"tiny": "最快速度,较低质量",diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.jsonindex ab1492246..48f0d7ab3 100644--- a/src/locales/zh-TW/translation.json+++ b/src/locales/zh-TW/translation.json@@ -1873,7 +1873,9 @@"descriptions": {"parakeet": {"parakeet_tdt_0_6b_v3": "快速多語言 ASR,自動語言偵測(25 種語言)",- "parakeet_unified_en_0_6b": "僅英語 ASR,最先進的準確度(平均 WER 5.91%)"+ "parakeet_unified_en_0_6b": "僅英語 ASR,最先進的準確度(平均 WER 5.91%)",+ "nemotron_speech_streaming_en_0_6b": "NVIDIA Nemotron 英語串流語音辨識",+ "nemotron_3_5_asr_streaming_0_6b": "NVIDIA Nemotron 多語言串流語音辨識,自動語言偵測(15 種語言)"},"whisper": {"tiny": "最快速,品質較低",diff --git a/src/models/ModelRegistry.ts b/src/models/ModelRegistry.tsindex f6639c48f..9745b8485 100644--- a/src/models/ModelRegistry.ts+++ b/src/models/ModelRegistry.ts@@ -99,6 +99,7 @@ export interface ParakeetModelInfo {sizeMb: number;language: string;supportedLanguages: string[];+ runtime?: "offline" | "online";recommended?: boolean;downloadUrl: string;extractDir: string;diff --git a/src/models/modelRegistryData.json b/src/models/modelRegistryData.jsonindex 3d75a1e8a..4620f7224 100644--- a/src/models/modelRegistryData.json+++ b/src/models/modelRegistryData.json@@ -47,6 +47,46 @@"downloadUrl": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-non-streaming.tar.bz2","extractDir": "sherpa-onnx-nemo-parakeet-unified-en-0.6b-int8-non-streaming","descriptionKey": "models.descriptions.parakeet.parakeet_unified_en_0_6b"+ },+ "nemotron-speech-streaming-en-0.6b": {+ "name": "Nemotron Speech Streaming EN 0.6B",+ "description": "English streaming ASR from NVIDIA Nemotron",+ "size": "632MB",+ "sizeMb": 632,+ "language": "en",+ "supportedLanguages": ["en"],+ "runtime": "online",+ "downloadUrl": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemotron-speech-streaming-en-0.6b-560ms-int8-2026-04-25.tar.bz2",+ "extractDir": "sherpa-onnx-nemotron-speech-streaming-en-0.6b-560ms-int8-2026-04-25",+ "descriptionKey": "models.descriptions.parakeet.nemotron_speech_streaming_en_0_6b"+ },+ "nemotron-3.5-asr-streaming-0.6b": {+ "name": "Nemotron 3.5 ASR Streaming 0.6B",+ "description": "Multilingual streaming ASR from NVIDIA Nemotron with auto language detection (15 languages)",+ "size": "650MB",+ "sizeMb": 650,+ "language": "multilingual",+ "supportedLanguages": [+ "en",+ "es",+ "fr",+ "it",+ "pt",+ "nl",+ "de",+ "tr",+ "ru",+ "ar",+ "hi",+ "ja",+ "ko",+ "vi",+ "uk"+ ],+ "runtime": "online",+ "downloadUrl": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11.tar.bz2",+ "extractDir": "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11",+ "descriptionKey": "models.descriptions.parakeet.nemotron_3_5_asr_streaming_0_6b"}},"diarizationModels": {diff --git a/src/utils/audioUtils.js b/src/utils/audioUtils.jsindex fb7d751be..b7c4677ae 100644--- a/src/utils/audioUtils.js+++ b/src/utils/audioUtils.js@@ -35,7 +35,17 @@ function pcm16ToWav(pcmBuffer, sampleRate = 16000, channels = 1) {return Buffer.concat([header, pcmBuffer]);}+function pcm16ToFloat32(pcmBuffer) {+ const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);+ const output = new Float32Array(input.length);+ for (let i = 0; i < input.length; i++) {+ output[i] = input[i] / 32768;+ }+ return output;+}+module.exports = {downsample24kTo16k,pcm16ToWav,+ pcm16ToFloat32,};diff --git a/test/helpers/parakeetModelRegistry.test.js b/test/helpers/parakeetModelRegistry.test.jsnew file mode 100644index 000000000..7c5dff655--- /dev/null+++ b/test/helpers/parakeetModelRegistry.test.js@@ -0,0 +1,44 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const modelData = require("../../src/models/modelRegistryData.json");+const { BINARIES } = require("../../scripts/download-sherpa-onnx");++test("Nemotron sherpa model uses the streaming runtime and bundled online server", () => {+ const model = modelData.parakeetModels["nemotron-speech-streaming-en-0.6b"];++ assert.equal(model.runtime, "online");+ assert.equal(+ model.extractDir,+ "sherpa-onnx-nemotron-speech-streaming-en-0.6b-560ms-int8-2026-04-25"+ );+ assert.match(+ model.downloadUrl,+ /sherpa-onnx-nemotron-speech-streaming-en-0\.6b-560ms-int8-2026-04-25\.tar\.bz2$/+ );+ assert.equal(model.language, "en");+ assert.deepEqual(model.supportedLanguages, ["en"]);++ for (const [platformArch, config] of Object.entries(BINARIES)) {+ assert.match(config.onlineBinaryPath, /sherpa-onnx-online-websocket-server/);+ assert.match(config.onlineOutputName, new RegExp(`^sherpa-onnx-online-ws-${platformArch}`));+ }+});++test("Nemotron 3.5 multilingual sherpa model uses the streaming runtime", () => {+ const model = modelData.parakeetModels["nemotron-3.5-asr-streaming-0.6b"];++ assert.equal(model.runtime, "online");+ assert.equal(+ model.extractDir,+ "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11"+ );+ assert.match(+ model.downloadUrl,+ /sherpa-onnx-nemotron-3\.5-asr-streaming-0\.6b-560ms-int8-2026-06-11\.tar\.bz2$/+ );+ assert.equal(model.language, "multilingual");+ assert.ok(model.supportedLanguages.length >= 15);+ assert.ok(model.supportedLanguages.includes("en"));+ assert.ok(model.supportedLanguages.includes("ja"));+});diff --git a/test/helpers/parakeetOnlineStream.test.js b/test/helpers/parakeetOnlineStream.test.jsnew file mode 100644index 000000000..18d02ba6e--- /dev/null+++ b/test/helpers/parakeetOnlineStream.test.js@@ -0,0 +1,146 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const { once } = require("node:events");++const { WebSocketServer } = require("ws");++const ParakeetWsServer = require("../../src/helpers/parakeetWsServer");+const { pcm16ToFloat32 } = require("../../src/utils/audioUtils");++// Mock sherpa online WS protocol: float32 binary frames in, JSON results out, "Done"/"Done!" handshake.+async function startMockOnlineServer({ onBinary, finalSegment = 0 }) {+ const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" });+ await once(wss, "listening");++ wss.on("connection", (socket) => {+ let binaryFrames = 0;+ socket.on("message", (data, isBinary) => {+ if (isBinary) {+ binaryFrames += 1;+ onBinary?.(socket, data, binaryFrames);+ return;+ }+ if (data.toString() === "Done") {+ socket.send(+ JSON.stringify({+ text: `final after ${binaryFrames} frames`,+ segment: finalSegment,+ is_final: true,+ })+ );+ socket.send("Done!");+ }+ });+ });++ return {+ port: wss.address().port,+ close: () => new Promise((resolve) => wss.close(resolve)),+ };+}++function onlineWsServerAt(port) {+ const server = new ParakeetWsServer();+ server.ready = true;+ server.process = { pid: 1 };+ server.port = port;+ server.modelRuntime = "online";+ return server;+}++test("online stream emits live updates and finish resolves with the final text", async () => {+ const mock = await startMockOnlineServer({+ onBinary: (socket, _data, frameCount) => {+ socket.send(+ JSON.stringify({ text: `partial ${frameCount}`, segment: 0, is_final: false })+ );+ },+ });++ try {+ const updates = [];+ const stream = onlineWsServerAt(mock.port).createOnlineStream({+ onUpdate: (text) => updates.push(text),+ });++ // Chunks sent before the socket opens must be queued, not dropped.+ stream.sendPcm16(Buffer.alloc(3200));+ stream.sendFloat32(pcm16ToFloat32(Buffer.alloc(3200)));++ const { text } = await stream.finish();++ assert.equal(text, "final after 2 frames");+ assert.ok(updates.includes("partial 1"));+ assert.ok(updates.includes("partial 2"));+ } finally {+ await mock.close();+ }+});++test("finalized segments accumulate across endpoints in live updates", async () => {+ let segment = 0;+ const mock = await startMockOnlineServer({+ finalSegment: 2,+ onBinary: (socket, _data, frameCount) => {+ // Simulate endpointing: every frame finalizes a segment.+ socket.send(+ JSON.stringify({ text: `segment ${frameCount}`, segment: segment++, is_final: true })+ );+ },+ });++ try {+ let lastUpdate = "";+ const stream = onlineWsServerAt(mock.port).createOnlineStream({+ onUpdate: (text) => {+ lastUpdate = text;+ },+ });++ stream.sendPcm16(Buffer.alloc(3200));+ stream.sendPcm16(Buffer.alloc(3200));++ const { text } = await stream.finish();++ assert.equal(text, "segment 1 segment 2 final after 2 frames");+ assert.equal(lastUpdate, text);+ } finally {+ await mock.close();+ }+});++test("createOnlineStream rejects offline-runtime sessions and dead servers", () => {+ const offline = onlineWsServerAt(1);+ offline.modelRuntime = "offline";+ assert.throws(() => offline.createOnlineStream(), /online-runtime/);++ const dead = onlineWsServerAt(1);+ dead.ready = false;+ assert.throws(() => dead.createOnlineStream(), /not running/);+});++test("abort closes the stream without waiting for the server", async () => {+ const mock = await startMockOnlineServer({});+ try {+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});+ stream.sendPcm16(Buffer.alloc(3200));+ stream.abort();+ const { text } = await stream.finish();+ assert.equal(text, "");+ } finally {+ await mock.close();+ }+});++test("pcm16ToFloat32 converts int16 samples to normalized float32", () => {+ const pcm = Buffer.alloc(8);+ pcm.writeInt16LE(0, 0);+ pcm.writeInt16LE(16384, 2);+ pcm.writeInt16LE(-16384, 4);+ pcm.writeInt16LE(-32768, 6);++ const floats = pcm16ToFloat32(pcm);++ assert.ok(floats instanceof Float32Array);+ assert.deepEqual(Array.from(floats), [0, 0.5, -0.5, -1]);+});diff --git a/test/helpers/parakeetWsResult.test.js b/test/helpers/parakeetWsResult.test.jsnew file mode 100644index 000000000..a583034fd--- /dev/null+++ b/test/helpers/parakeetWsResult.test.js@@ -0,0 +1,56 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const { parseOfflineMessage, parseOnlineMessages } = require("../../src/helpers/parakeetWsResult");++test("offline sherpa message extracts JSON text", () => {+ assert.equal(parseOfflineMessage(JSON.stringify({ text: "hello world" })), "hello world");+ assert.equal(parseOfflineMessage("2026"), "2026");+});++test("online sherpa messages keep finalized segments and latest partial", () => {+ assert.equal(+ parseOnlineMessages([+ JSON.stringify({ text: "hello", is_final: false, segment: 0 }),+ JSON.stringify({ text: "hello world", is_final: true, segment: 0 }),+ JSON.stringify({ text: "again", is_final: false, segment: 1 }),+ ]),+ "hello world again"+ );+});++test("online sherpa parser keeps numeric final text as text", () => {+ assert.equal(+ parseOnlineMessages([JSON.stringify({ text: "2026", is_final: true, segment: 0 })]),+ "2026"+ );+});++test("online sherpa parser ignores JSON scalar control messages", () => {+ assert.equal(+ parseOnlineMessages([+ "null",+ "true",+ JSON.stringify({ text: "hello", is_final: true, segment: 0 }),+ ]),+ "hello"+ );+});++test("online sherpa parser keeps latest partial before ignored trailing messages", () => {+ assert.equal(+ parseOnlineMessages([+ JSON.stringify({ text: "hello", is_final: false, segment: 0 }),+ "null",+ JSON.stringify({ text: "", is_final: false, segment: 0 }),+ ]),+ "hello"+ );+});++test("online sherpa parser tolerates non-string text payloads", () => {+ assert.equal(+ parseOnlineMessages([JSON.stringify({ text: 2026, is_final: true, segment: 0 })]),+ "2026"+ );+});