Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

refactor(voice): collect microphone frames with a new worklet sized from the sample rate

Luis Ortiz09951ac1e632
diff --git a/apps/desktop/src/renderer/narration.ts b/apps/desktop/src/renderer/narration.ts
index a070b3a..1665c03 100644
--- a/apps/desktop/src/renderer/narration.ts
+++ b/apps/desktop/src/renderer/narration.ts
@@ -141,7 +141,7 @@ export class DesktopNarration {
this.report({ phase: 'error', reason: 'missing-api-key' });
return null;
}
- const mic = new MicRecorder(assetUrl('pcm-processor.js'), {
+ const mic = new MicRecorder(assetUrl('mic-frames.js'), {
onEpoch: () => undefined,
onLevel: (level, speaking) => window.mimik.capture.narration({ level, speaking }),
onStreamEnded: () => {
diff --git a/apps/extension/public/mic-frames.js b/apps/extension/public/mic-frames.js
new file mode 100644
index 0000000..2dbf0a9
--- /dev/null
+++ b/apps/extension/public/mic-frames.js
@@ -0,0 +1,36 @@
+class MicFrameCollector extends AudioWorkletProcessor {
+ constructor(options) {
+ super();
+ const frameMs = options?.processorOptions?.frameMs ?? 250;
+ this.pending = new Float32Array(Math.max(1, Math.round((sampleRate * frameMs) / 1000)));
+ this.filled = 0;
+ }
+
+ process(inputs) {
+ const mono = inputs[0]?.[0];
+ if (!mono) return true;
+ let read = 0;
+ while (read < mono.length) {
+ const room = this.pending.length - this.filled;
+ const take = Math.min(room, mono.length - read);
+ this.pending.set(mono.subarray(read, read + take), this.filled);
+ this.filled += take;
+ read += take;
+ if (this.filled === this.pending.length) this.flush();
+ }
+ return true;
+ }
+
+ flush() {
+ const bytes = new ArrayBuffer(this.pending.length * 2);
+ const view = new DataView(bytes);
+ for (let i = 0; i < this.pending.length; i++) {
+ const level = Math.max(-1, Math.min(1, this.pending[i]));
+ view.setInt16(i * 2, Math.round(level * 0x7fff), true);
+ }
+ this.port.postMessage(bytes, [bytes]);
+ this.filled = 0;
+ }
+}
+
+registerProcessor('mic-frames', MicFrameCollector);
diff --git a/apps/extension/public/pcm-processor.js b/apps/extension/public/pcm-processor.js
deleted file mode 100644
index 522a0e8..0000000
--- a/apps/extension/public/pcm-processor.js
+++ /dev/null
@@ -1,25 +0,0 @@
-const CHUNK = 4800;
-
-class PCMProcessor extends AudioWorkletProcessor {
- constructor() {
- super();
- this.buffer = new Int16Array(CHUNK);
- this.index = 0;
- }
-
- process(inputs) {
- const channel = inputs[0] && inputs[0][0];
- if (!channel) return true;
- for (const sample of channel) {
- this.buffer[this.index++] = sample > 1 ? 32767 : sample < -1 ? -32767 : (sample * 32767) | 0;
- if (this.index >= CHUNK) {
- this.port.postMessage(this.buffer.buffer, [this.buffer.buffer]);
- this.buffer = new Int16Array(CHUNK);
- this.index = 0;
- }
- }
- return true;
- }
-}
-
-registerProcessor('pcm-processor', PCMProcessor);
diff --git a/apps/extension/src/lib/voice/__tests__/mic-frames.test.ts b/apps/extension/src/lib/voice/__tests__/mic-frames.test.ts
new file mode 100644
index 0000000..177480a
--- /dev/null
+++ b/apps/extension/src/lib/voice/__tests__/mic-frames.test.ts
@@ -0,0 +1,57 @@
+import { beforeAll, describe, expect, it, vi } from 'vitest';
+
+type Collector = {
+ port: { postMessage: ReturnType<typeof vi.fn> };
+ process(inputs: Float32Array[][]): boolean;
+};
+type CollectorClass = new (options?: { processorOptions?: { frameMs?: number } }) => Collector;
+
+let Registered: CollectorClass;
+let registeredName = '';
+
+beforeAll(async () => {
+ Object.assign(globalThis, {
+ sampleRate: 16000,
+ AudioWorkletProcessor: class {
+ port = { postMessage: vi.fn() };
+ },
+ registerProcessor: (name: string, processor: CollectorClass) => {
+ registeredName = name;
+ Registered = processor;
+ },
+ });
+ await import(new URL('../../../../public/mic-frames.js', import.meta.url).href);
+});
+
+function posted(collector: Collector): Int16Array[] {
+ return collector.port.postMessage.mock.calls.map(([buffer]) => new Int16Array(buffer as ArrayBuffer));
+}
+
+describe('mic-frames worklet', () => {
+ it('registers under the name the recorder asks for', () => {
+ expect(registeredName).toBe('mic-frames');
+ });
+
+ it('sends one frame per frameMs of audio, sized from the sample rate', () => {
+ const collector = new Registered({ processorOptions: { frameMs: 1 } });
+ collector.process([[new Float32Array(40)]]);
+
+ expect(posted(collector).map((frame) => frame.length)).toEqual([16, 16]);
+ });
+
+ it('clamps and scales samples symmetrically into 16-bit integers', () => {
+ const collector = new Registered({ processorOptions: { frameMs: 0.375 } });
+ collector.process([[Float32Array.from([2, 1, 0.5, 0, -0.5, -1])]]);
+
+ expect([...posted(collector)[0]!]).toEqual([32767, 32767, 16384, 0, -16383, -32767]);
+ });
+
+ it('carries a frame across render quanta and ignores an empty input', () => {
+ const collector = new Registered({ processorOptions: { frameMs: 0.25 } });
+ collector.process([[Float32Array.from([0.1, 0.2, 0.3])]]);
+ collector.process([[]]);
+ collector.process([[Float32Array.from([0.4, 0.5])]]);
+
+ expect(posted(collector).map((frame) => frame.length)).toEqual([4]);
+ });
+});
diff --git a/apps/extension/src/lib/voice/create-voice-host.ts b/apps/extension/src/lib/voice/create-voice-host.ts
index a67e824..207ba2c 100644
--- a/apps/extension/src/lib/voice/create-voice-host.ts
+++ b/apps/extension/src/lib/voice/create-voice-host.ts
@@ -46,7 +46,7 @@ export function createVoiceHost(): VoiceHost {
let retained: VoiceRecording | null = null;
const slicer = new NarrationSlicer();
- const recorder = new MicRecorder(getExtensionURL('/pcm-processor.js'), {
+ const recorder = new MicRecorder(getExtensionURL('/mic-frames.js'), {
onEpoch: (audioEpochMs) =>
emit(
voiceMessage<VoiceEpochEvent>({
diff --git a/packages/core/src/capture/voice/mic-recorder.ts b/packages/core/src/capture/voice/mic-recorder.ts
index c447752..8eb1fec 100644
--- a/packages/core/src/capture/voice/mic-recorder.ts
+++ b/packages/core/src/capture/voice/mic-recorder.ts
@@ -3,7 +3,8 @@ import { SPEECH_RMS_THRESHOLD } from './energy-gate';
import { openStream } from './open-stream';
const TARGET_SAMPLE_RATE = 16000;
-const PCM_WORKLET_NAME = 'pcm-processor';
+const MIC_FRAMES_WORKLET = 'mic-frames';
+const FRAME_MS = 250;
const ANALYSER_FFT_SIZE = 2048;
const LEVEL_FLOOR_DB = -60;
@@ -83,7 +84,7 @@ export class MicRecorder {
this.analyser = context.createAnalyser();
this.analyser.fftSize = ANALYSER_FFT_SIZE;
this.analyserFrame = new Float32Array(this.analyser.fftSize);
- this.worklet = new AudioWorkletNode(context, PCM_WORKLET_NAME);
+ this.worklet = new AudioWorkletNode(context, MIC_FRAMES_WORKLET, { processorOptions: { frameMs: FRAME_MS } });
this.worklet.port.onmessage = (event: MessageEvent<ArrayBuffer>) => this.appendFrame(event.data);
this.source.connect(this.analyser);