Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

fix(speech): start smart-whisper's model params from whisper.cpp's defaults

Dev Goswami8455a0eccbf3
diff --git a/CLAUDE.md b/CLAUDE.md
index 3e8e3bdaa..0369261a6 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -9893,6 +9893,16 @@ Voice-to-text input captured via microphone, turned into terminal text via on-de
SIMD flags in ggml), a GPU backend (this smart-whisper has no Vulkan build option), cancelling a
running transcription, and the Server Edition (installed with `--ignore-scripts`, so it keeps the
baseline; the startup log line `[speech] whisper binary: …` says which binary loaded).
+- **smart-whisper's model params are patched to start from whisper.cpp's defaults** (second patch in
+ `scripts/patch-smart-whisper.mjs`, marker `NODETERM-PATCH(smart-whisper-ctx-params)`, every
+ platform). Its `LoadModel` declared `whisper_context_params params;` and set only `use_gpu`, so DTW,
+ `gpu_device`, `flash_attn` and the rest were uninitialized stack memory and the result depended on
+ what was there. MEASURED on an M5 Pro, Electron 42, large-v3-turbo: the shipped binary crashed in 4
+ of 4 runs outside the app (`aheads_masks_init` then a crash in `whisper_free_state`, because garbage
+ switched DTW on with large-v3's alignment heads) and took 7-10 s per dictation inside it; with the
+ defaults it took 0.56 s for a 2.7 s clip and 0.93 s for 21 s, no crash. A newer whisper.cpp is
+ faster again (0.22 s / 0.43 s, measured the same way) but needs a build change on every platform;
+ that is a separate follow-up.
## Packaging & auto-update
diff --git a/scripts/patch-smart-whisper.d.mts b/scripts/patch-smart-whisper.d.mts
index 25b15f9b2..c6c0e1c51 100644
--- a/scripts/patch-smart-whisper.d.mts
+++ b/scripts/patch-smart-whisper.d.mts
@@ -3,4 +3,5 @@
export declare const PATCH_MARKER: string
export declare const VARIANT_TARGET: string
export declare const SIMD_FLAGS: readonly string[]
+export declare const CTX_PARAMS_MARKER: string
export declare function patchSmartWhisper(dir: string): 'patched' | 'already-patched'
diff --git a/scripts/patch-smart-whisper.mjs b/scripts/patch-smart-whisper.mjs
index dbf65eaca..2bea88354 100644
--- a/scripts/patch-smart-whisper.mjs
+++ b/scripts/patch-smart-whisper.mjs
@@ -28,7 +28,18 @@
* no-op when nothing sets the global.
* - Runs before electron-rebuild in `postinstall` / `rebuild`, so the native build sees it.
*
- * GUARD: src/core/speech/smart-whisper-patch.test.ts asserts the marker is present in the
+ * SECOND PATCH: initialized context params (every platform)
+ * smart-whisper's `LoadModel` (src/binding/model.cc) declares `whisper_context_params params;`
+ * and sets only `use_gpu`, so every other field (dtw_token_timestamps, dtw_aheads_preset,
+ * gpu_device, flash_attn, ...) is uninitialized stack memory. What it holds decides the outcome:
+ * with DTW switched on by garbage, whisper_init_state() fails for large-v3-turbo ("tried to set
+ * alignment head on text layer 14, but model only has 4 text layers") and the app aborts in
+ * whisper_free_state. Measured on an M5 Pro, Electron 42, large-v3-turbo, 2.7 s and 21 s clips:
+ * the shipped binary crashed in 4 of 4 runs outside the app and took 7-10 s per dictation inside
+ * it; with the params initialized it took 0.56 s and 0.93 s, no crash. The fix starts from
+ * whisper_context_default_params(), exactly what whisper.cpp's own examples do.
+ *
+ * GUARD: src/core/speech/smart-whisper-patch.test.ts asserts both markers are present in the
* installed sources. If it is red, your node_modules is unpatched — run `npm run rebuild`.
*/
@@ -65,15 +76,34 @@ var module2 = (() => {
const GYP_FIND = ' }\n ],\n}'
+export const CTX_PARAMS_MARKER = 'NODETERM-PATCH(smart-whisper-ctx-params)'
+
+const PARAMS_FIND = ' whisper_context_params params;\n params.use_gpu'
+
+const PARAMS_REPLACE =
+ ` // ${CTX_PARAMS_MARKER}: start from the defaults; the bare declaration left every\n` +
+ ` // other field (DTW, gpu_device, flash_attn, ...) as uninitialized stack memory.\n` +
+ ' whisper_context_params params = whisper_context_default_params();\n' +
+ ' params.use_gpu'
+
/** Patches the smart-whisper package in `dir`. Returns 'patched' | 'already-patched'. */
export function patchSmartWhisper(dir) {
const gypPath = path.join(dir, 'binding.gyp')
const jsPath = path.join(dir, 'dist', 'index.js')
const gyp = fs.readFileSync(gypPath, 'utf8')
const js = fs.readFileSync(jsPath, 'utf8')
+ const modelPath = path.join(dir, 'src', 'binding', 'model.cc')
+ if (!fs.existsSync(modelPath)) fail('src/binding/model.cc', 'not found')
+ const model = fs.readFileSync(modelPath, 'utf8')
const gypDone = gyp.includes(PATCH_MARKER)
const jsDone = js.includes(PATCH_MARKER)
- if (gypDone && jsDone) return 'already-patched'
+ const modelDone = model.includes(CTX_PARAMS_MARKER)
+ if (gypDone && jsDone && modelDone) return 'already-patched'
+
+ if (!modelDone) {
+ expectOnce(model, PARAMS_FIND, 'model.cc context params')
+ fs.writeFileSync(modelPath, model.replace(PARAMS_FIND, PARAMS_REPLACE))
+ }
if (!jsDone) {
expectOnce(js, LOADER_FIND, 'dist/index.js binary require')
diff --git a/src/core/speech/__fixtures__/smart-whisper-0.8.1/model.cc b/src/core/speech/__fixtures__/smart-whisper-0.8.1/model.cc
new file mode 100644
index 000000000..902f8888c
--- /dev/null
+++ b/src/core/speech/__fixtures__/smart-whisper-0.8.1/model.cc
@@ -0,0 +1,145 @@
+#include "model.h"
+
+class LoadModelWorker : public PromiseWorker {
+ public:
+ LoadModelWorker(Napi::Env &env, const std::string &model_path,
+ struct whisper_context_params params)
+ : PromiseWorker(env), model_path(model_path), params(params) {}
+
+ void Execute() override {
+ context = whisper_init_from_file_with_params_no_state(model_path.c_str(), params);
+ if (context == nullptr) {
+ SetError("Failed to initialize whisper context");
+ }
+ whisper_print_timings(context);
+ }
+
+ void OnOK() override {
+ Napi::HandleScope scope(Env());
+ auto handle = Napi::External<whisper_context>::New(Env(), context);
+ auto constructor = Env().GetInstanceData<Napi::FunctionReference>();
+ auto model = constructor->New({handle});
+
+ promise.Resolve(model);
+ }
+
+ private:
+ std::string model_path;
+ struct whisper_context_params params;
+ whisper_context *context;
+};
+
+class FreeModelWorker : public PromiseWorker {
+ public:
+ FreeModelWorker(Napi::Env &env, whisper_context *context)
+ : PromiseWorker(env), context(context) {}
+
+ void Execute() override { whisper_free(context); }
+
+ void OnOK() override {
+ Napi::HandleScope scope(Env());
+ promise.Resolve(Env().Undefined());
+ }
+
+ private:
+ whisper_context *context;
+};
+
+Napi::Object WhisperModel::Init(Napi::Env env, Napi::Object exports) {
+ Napi::Function func = DefineClass(
+ env, "WhisperModel",
+ {
+ StaticMethod<&WhisperModel::Load>(
+ "load", static_cast<napi_property_attributes>(napi_writable | napi_configurable)),
+ InstanceMethod<&WhisperModel::Free>(
+ "free", static_cast<napi_property_attributes>(napi_writable | napi_configurable)),
+ InstanceAccessor(
+ "freed", &WhisperModel::GetFreed, nullptr,
+ static_cast<napi_property_attributes>(napi_enumerable | napi_configurable)),
+ InstanceAccessor(
+ "handle", &WhisperModel::GetHandle, nullptr,
+ static_cast<napi_property_attributes>(napi_enumerable | napi_configurable)),
+ });
+
+ auto constructor = new Napi::FunctionReference();
+ *constructor = Napi::Persistent(func);
+ env.SetInstanceData<Napi::FunctionReference>(constructor);
+
+ exports.Set("WhisperModel", func);
+ return exports;
+}
+
+WhisperModel::WhisperModel(const Napi::CallbackInfo &info) : Napi::ObjectWrap<WhisperModel>(info) {
+ Napi::Env env = info.Env();
+ Napi::HandleScope scope(env);
+
+ if (info.Length() != 1) {
+ Napi::TypeError::New(env, "Wrong number of arguments").ThrowAsJavaScriptException();
+ return;
+ }
+
+ whisper_context *context = info[0].As<Napi::External<whisper_context>>().Data();
+ this->context = context;
+}
+
+void WhisperModel::Finalize(Napi::Env env) {
+ if (context != nullptr) {
+ whisper_free(context);
+ context = nullptr;
+ }
+}
+
+Napi::Value WhisperModel::Load(const Napi::CallbackInfo &info) {
+ Napi::Env env = info.Env();
+
+ if (info.Length() < 1 || info.Length() > 2) {
+ Napi::TypeError::New(env, "Wrong number of arguments").ThrowAsJavaScriptException();
+ return env.Null();
+ }
+
+ std::string model_path = info[0].As<Napi::String>();
+
+ whisper_context_params params;
+ params.use_gpu = info.Length() == 2 ? info[1].As<Napi::Boolean>() : true;
+
+ auto worker = new LoadModelWorker(env, model_path, params);
+ worker->Queue();
+
+ return worker->Promise();
+}
+
+Napi::Value WhisperModel::Free(const Napi::CallbackInfo &info) {
+ Napi::Env env = info.Env();
+
+ if (info.Length() != 0) {
+ Napi::TypeError::New(env, "Wrong number of arguments").ThrowAsJavaScriptException();
+ return env.Null();
+ }
+
+ if (context == nullptr) {
+ auto deferred = Napi::Promise::Deferred::New(env);
+ deferred.Resolve(env.Undefined());
+ return deferred.Promise();
+ } else {
+ auto worker = new FreeModelWorker(env, context);
+ context = nullptr;
+ worker->Queue();
+ return worker->Promise();
+ }
+}
+
+Napi::Value WhisperModel::GetFreed(const Napi::CallbackInfo &info) {
+ Napi::Env env = info.Env();
+
+ return Napi::Boolean::New(env, context == nullptr);
+}
+
+Napi::Value WhisperModel::GetHandle(const Napi::CallbackInfo &info) {
+ Napi::Env env = info.Env();
+
+ if (context == nullptr) {
+ return env.Null();
+ }
+
+ return Napi::External<whisper_context>::New(env, context);
+}
diff --git a/src/core/speech/smart-whisper-patch.test.ts b/src/core/speech/smart-whisper-patch.test.ts
index 478142637..c72e01624 100644
--- a/src/core/speech/smart-whisper-patch.test.ts
+++ b/src/core/speech/smart-whisper-patch.test.ts
@@ -2,7 +2,13 @@ import { describe, expect, it } from 'vitest'
import fs from 'node:fs'
import path from 'node:path'
import { createRequire } from 'node:module'
-import { PATCH_MARKER, SIMD_FLAGS, VARIANT_TARGET, patchSmartWhisper } from '../../../scripts/patch-smart-whisper.mjs'
+import {
+ CTX_PARAMS_MARKER,
+ PATCH_MARKER,
+ SIMD_FLAGS,
+ VARIANT_TARGET,
+ patchSmartWhisper
+} from '../../../scripts/patch-smart-whisper.mjs'
import { WHISPER_SIMD_FLAGS } from './speech-service'
import { testTmpDir } from '../test-tmp'
@@ -14,7 +20,7 @@ import { testTmpDir } from '../test-tmp'
*/
const PKG = path.resolve(__dirname, '../../../node_modules/smart-whisper')
const installed = fs.existsSync(path.join(PKG, 'binding.gyp'))
-/** Pristine smart-whisper 0.8.1 binding.gyp + the loader lines of its dist/index.js (dist-index.js). */
+/** Pristine smart-whisper 0.8.1 binding.gyp, src/binding/model.cc, and its dist/index.js (dist-index.js). */
const FIXTURE = path.resolve(__dirname, '__fixtures__/smart-whisper-0.8.1')
function copyPackage(): string {
@@ -23,6 +29,8 @@ function copyPackage(): string {
fs.copyFileSync(path.join(FIXTURE, 'binding.gyp'), path.join(dir, 'binding.gyp'))
// Stored flat: the repo's .gitignore drops every dist/ directory.
fs.copyFileSync(path.join(FIXTURE, 'dist-index.js'), path.join(dir, 'dist', 'index.js'))
+ fs.mkdirSync(path.join(dir, 'src', 'binding'), { recursive: true })
+ fs.copyFileSync(path.join(FIXTURE, 'model.cc'), path.join(dir, 'src', 'binding', 'model.cc'))
return dir
}
@@ -31,6 +39,27 @@ describe('patch-smart-whisper', () => {
const howto = 'node_modules is unpatched — run `npm run rebuild`.'
expect(fs.readFileSync(path.join(PKG, 'binding.gyp'), 'utf8'), howto).toContain(PATCH_MARKER)
expect(fs.readFileSync(path.join(PKG, 'dist', 'index.js'), 'utf8'), howto).toContain(PATCH_MARKER)
+ expect(fs.readFileSync(path.join(PKG, 'src', 'binding', 'model.cc'), 'utf8'), howto).toContain(CTX_PARAMS_MARKER)
+ })
+
+ it('starts the model context params from whisper.cpp defaults and is idempotent', () => {
+ // The pristine declaration leaves every field but use_gpu as stack garbage (DTW among them).
+ const dir = copyPackage()
+ const modelPath = path.join(dir, 'src', 'binding', 'model.cc')
+ expect(fs.readFileSync(modelPath, 'utf8')).toMatch(/^\s*whisper_context_params params;$/m)
+ expect(patchSmartWhisper(dir)).toBe('patched')
+ const model = fs.readFileSync(modelPath, 'utf8')
+ expect(model).toContain('whisper_context_params params = whisper_context_default_params();')
+ expect(model).not.toMatch(/^\s*whisper_context_params params;$/m)
+ expect(model).toContain('params.use_gpu = info.Length() == 2')
+ expect(patchSmartWhisper(dir)).toBe('already-patched')
+ expect(fs.readFileSync(modelPath, 'utf8')).toBe(model)
+ })
+
+ it('refuses a model.cc whose params declaration moved', () => {
+ const dir = copyPackage()
+ fs.writeFileSync(path.join(dir, 'src', 'binding', 'model.cc'), '// rewritten upstream\n')
+ expect(() => patchSmartWhisper(dir)).toThrow(/anchor "model.cc context params"/)
})
it('builds for exactly the CPU flags the runtime checks', () => {