Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

fix(voice): flush captured speech before finishing recognition

BlueSkyID6666ae4fcb9c415
diff --git a/src/renderer/modules/voice-input.js b/src/renderer/modules/voice-input.js
index a22a78f4..35c51f2c 100644
--- a/src/renderer/modules/voice-input.js
+++ b/src/renderer/modules/voice-input.js
@@ -37,6 +37,7 @@
this._connectTimer = null;
this._finishTimer = null;
this._closeEmitted = false;
+ this._captureFlush = null;
}
emit(name, ...args) {
@@ -199,15 +200,22 @@
if (this._audioCtx.audioWorklet && typeof window.AudioWorkletNode === 'function') {
let url = '';
try {
- const code = `class C extends AudioWorkletProcessor{constructor(){super();this.n=1024;this.b=new Float32Array(this.n);this.i=0;this.port.onmessage=e=>{const d=e.data||{};if(typeof d.n==='number'&&d.n>0){this.n=d.n|0;this.b=new Float32Array(this.n);this.i=0;}};}process(inp,out){const a=inp&&inp[0]&&inp[0][0];const o=out&&out[0]&&out[0][0];if(o){for(let i=0;i<o.length;i++)o[i]=0;}if(!a||a.length===0)return true;for(let i=0;i<a.length;i++){this.b[this.i++]=a[i];if(this.i>=this.n){const c=new Float32Array(this.b);this.port.postMessage({b:c},[c.buffer]);this.i=0;}}return true;}}registerProcessor('orkas-asr-capture',C);`;
+ const code = `class C extends AudioWorkletProcessor{constructor(){super();this.n=1024;this.b=new Float32Array(this.n);this.i=0;this.stopped=false;this.port.onmessage=e=>{const d=e.data||{};if(d.flush){this.stopped=true;if(this.i){const c=this.b.slice(0,this.i);this.port.postMessage({b:c},[c.buffer]);this.i=0;}this.port.postMessage({flushed:true});return;}if(typeof d.n==='number'&&d.n>0){this.n=d.n|0;this.b=new Float32Array(this.n);this.i=0;}};}process(inp,out){if(this.stopped)return false;const a=inp&&inp[0]&&inp[0][0];const o=out&&out[0]&&out[0][0];if(o){for(let i=0;i<o.length;i++)o[i]=0;}if(!a||a.length===0)return true;for(let i=0;i<a.length;i++){this.b[this.i++]=a[i];if(this.i>=this.n){const c=new Float32Array(this.b);this.port.postMessage({b:c},[c.buffer]);this.i=0;}}return true;}}registerProcessor('orkas-asr-capture',C);`;
const blob = new Blob([code], { type: 'application/javascript' });
url = URL.createObjectURL(blob);
this._workletModuleURL = url;
await this._audioCtx.audioWorklet.addModule(url);
+ if (!this._running) return;
this._workletNode = new AudioWorkletNode(this._audioCtx, 'orkas-asr-capture', {
numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1],
});
this._workletNode.port.onmessage = (ev) => {
+ if (ev.data && ev.data.flushed) {
+ const pending = this._captureFlush;
+ this._captureFlush = null;
+ if (pending) { clearTimeout(pending.timer); pending.resolve(true); }
+ return;
+ }
const buf = ev.data && ev.data.b;
if (!buf) return;
const f32 = buf instanceof Float32Array ? buf : new Float32Array(buf);
@@ -252,6 +260,9 @@
}
teardownCapture() {
+ const pending = this._captureFlush;
+ this._captureFlush = null;
+ if (pending) { clearTimeout(pending.timer); pending.resolve(false); }
try { if (this._workletNode) this._workletNode.port.onmessage = null; } catch {}
try { if (this._scriptNode) this._scriptNode.onaudioprocess = null; } catch {}
try { if (this._workletNode) this._workletNode.disconnect(); } catch {}
@@ -295,20 +306,45 @@
return u8;
}
- stop() {
- this.teardownCapture();
+ async stop() {
+ if (this._pendingClose) return;
this._running = false;
this._pendingClose = true;
+ // Release the device immediately, but drain the worklet's partial packet
+ // before __END__. Otherwise up to 200ms of the final spoken word is lost.
+ if (this._mediaStream) {
+ try { this._mediaStream.getTracks().forEach((tr) => { try { tr.stop(); } catch {} }); } catch {}
+ this._mediaStream = null;
+ }
if (!this._ws) {
+ this.teardownCapture();
this.emitCloseOnce();
return;
}
this._cancelledBeforeOpen = this._ws.readyState === WebSocket.CONNECTING;
- try { this.sendAudio(new Uint8Array(0), true); } catch {}
// Bound finalization without dropping slower terminal responses immediately.
try {
this._finishTimer = setTimeout(() => { if (this._pendingClose) { try { this.close(); } catch {} } }, 10_000);
} catch {}
+ if (this._workletNode) {
+ const flushed = await new Promise((resolve) => {
+ const timer = setTimeout(() => {
+ this._captureFlush = null;
+ resolve(false);
+ }, 500);
+ this._captureFlush = { resolve, timer };
+ try { this._workletNode.port.postMessage({ flush: true }); }
+ catch { clearTimeout(timer); this._captureFlush = null; resolve(false); }
+ });
+ if (!this._ws) return;
+ if (!flushed) {
+ this.emit('onError', Object.assign(new Error(), { code: 'incomplete_transcript' }));
+ this.close();
+ return;
+ }
+ }
+ this.teardownCapture();
+ try { this.sendAudio(new Uint8Array(0), true); } catch {}
}
close() {
diff --git a/test/e2e/voice_input_e2e_lifecycle.spec.ts b/test/e2e/voice_input_e2e_lifecycle.spec.ts
index 98638ea2..53fdf8d5 100644
--- a/test/e2e/voice_input_e2e_lifecycle.spec.ts
+++ b/test/e2e/voice_input_e2e_lifecycle.spec.ts
@@ -52,7 +52,12 @@ test.describe('API-key voice input', () => {
createGain = () => ({ ...node(), gain: { value: 0 } });
close = async () => {};
};
- w.AudioWorkletNode = class { port = { onmessage: null, postMessage() {} }; connect() {} disconnect() {} };
+ w.AudioWorkletNode = class {
+ port = { onmessage: null as any, postMessage: (data: any) => {
+ if (data.flush) queueMicrotask(() => this.port.onmessage?.({ data: { flushed: true } }));
+ } };
+ connect() {} disconnect() {}
+ };
});
// Electron's CDP bridge does not intercept WebSockets through Playwright.
// Use a real loopback peer; only the transport constructor is redirected.
diff --git a/test/renderer/voice-input.test.ts b/test/renderer/voice-input.test.ts
index 74ecde5b..06ddf012 100644
--- a/test/renderer/voice-input.test.ts
+++ b/test/renderer/voice-input.test.ts
@@ -104,6 +104,75 @@ describe('voice input user journey', () => {
expect(h.button.dataset.voicePhase).toBe('idle');
expect(h.invoke.mock.calls.map(c => c[0])).not.toContain('groupChat.send');
});
+ it.each(['success', 'unresponsive', 'cancelled'])(
+ 'drains the speech tail or cancels incomplete capture safely: %s', async (scenario) => {
+ vi.useFakeTimers();
+ const h = load();
+ let processor: any, Processor: any, blob: Blob;
+ let mainPort: any;
+ class WorkletProcessor {
+ port = { onmessage: null as any, postMessage: (data: any) => {
+ queueMicrotask(() => mainPort.onmessage?.({ data }));
+ } };
+ }
+ class WorkletNode {
+ port = mainPort = { onmessage: null as any, postMessage: (data: any) => {
+ if (data.flush && scenario !== 'success') return;
+ queueMicrotask(() => processor.port.onmessage?.({ data }));
+ } };
+ constructor() { processor = new Processor(); }
+ connect() {}
+ disconnect() {}
+ }
+ const BaseContext = h.context.window.AudioContext;
+ h.context.window.AudioContext = class extends BaseContext {
+ sampleRate = 16000;
+ audioWorklet = { addModule: async () => {
+ vm.runInNewContext(await blob.text(), {
+ AudioWorkletProcessor: WorkletProcessor,
+ registerProcessor: (_: string, value: any) => { Processor = value; },
+ });
+ } };
+ };
+ h.context.window.AudioWorkletNode = h.context.AudioWorkletNode = WorkletNode;
+ h.context.Blob = Blob;
+ h.context.URL = class extends URL {
+ static createObjectURL(value: Blob) { blob = value; return 'blob:voice-test'; }
+ static revokeObjectURL() {}
+ };
+ await h.button.dispatch('click'); await flush();
+ const socket = h.socket(); socket.open(); await flush();
+ // At 16kHz, this is 200ms of complete packets plus a 40ms speech tail.
+ for (let i = 0; i < 30; i++) {
+ processor.process([[new Float32Array(128).fill(i < 25 ? .25 : -.5)]], [[new Float32Array(128)]]);
+ }
+ await flush();
+ await h.button.dispatch('click'); await flush();
+ expect(h.track.stop).toHaveBeenCalledTimes(1);
+ if (scenario !== 'success') {
+ expect(socket.send.mock.calls.some((call: any[]) => call[0] === '__END__')).toBe(false);
+ if (scenario === 'cancelled') h.context.window.VoiceInput.cancel();
+ await vi.advanceTimersByTimeAsync(500);
+ await flush();
+ expect(socket.send.mock.calls.some((call: any[]) => call[0] === '__END__')).toBe(false);
+ expect(h.button.dataset.voicePhase).toBe('idle');
+ expect(h.input.value).toBe('Draft:');
+ if (scenario === 'unresponsive') {
+ expect(h.context.uiAlert).toHaveBeenCalledWith('chat.mic_incomplete');
+ } else {
+ expect(h.context.uiAlert).not.toHaveBeenCalled();
+ }
+ return;
+ }
+ const sent = socket.send.mock.calls.map((call: any[]) => call[0]);
+ const audio = sent.filter((data: any) => ArrayBuffer.isView(data) && data.byteLength > 0);
+ expect(audio.reduce((n: number, data: any) => n + data.byteLength, 0)).toBe(7680);
+ expect(Array.from(audio.at(-1).slice(-2))).toEqual([0, 192]);
+ expect(sent.filter((value: any) => value === '__END__')).toHaveLength(1);
+ expect(sent.at(-1)).toBe('__END__');
+ socket.result('complete spoken note', true);
+ expect(h.input.value).toBe('Draft: complete spoken note');
+ });
it('cancels a pending permission request without resurrecting recording', async () => {
vi.useFakeTimers();
const h = load(); let grant: any;
@@ -115,6 +184,28 @@ describe('voice input user journey', () => {
expect(h.socket()).toBeUndefined();
expect(h.button.dataset.voicePhase).toBe('idle');
});
+ it('does not resurrect capture when stopped during worklet initialization', async () => {
+ vi.useFakeTimers();
+ const h = load();
+ let ready: () => void = () => {};
+ const BaseContext = h.context.window.AudioContext;
+ h.context.window.AudioContext = class extends BaseContext {
+ audioWorklet = { addModule: () => new Promise<void>((resolve) => { ready = resolve; }) };
+ };
+ h.context.Blob = Blob;
+ const WorkletNode = vi.fn();
+ h.context.window.AudioWorkletNode = h.context.AudioWorkletNode = WorkletNode;
+ await h.button.dispatch('click'); await flush();
+ const socket = h.socket(); socket.open(); await flush();
+ await h.button.dispatch('click'); await flush();
+ ready(); await flush();
+ expect(WorkletNode).not.toHaveBeenCalled();
+ expect(h.track.stop).toHaveBeenCalledTimes(1);
+ expect(h.context.uiAlert).not.toHaveBeenCalled();
+ socket.result('review this note', true);
+ expect(h.input.value).toBe('Draft: review this note');
+ expect(h.button.dataset.voicePhase).toBe('idle');
+ });
it('keeps the selected draft untouched when recognition produces no speech', async () => {
vi.useFakeTimers();
const h = load();