π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
refactor(voice): collect microphone frames with a new worklet sized from the sample rate
Luis Ortiz09951ac1e632
diff --git a/apps/desktop/src/renderer/narration.ts b/apps/desktop/src/renderer/narration.tsindex a070b3a..1665c03 100644--- a/apps/desktop/src/renderer/narration.ts+++ b/apps/desktop/src/renderer/narration.ts@@ -141,7 +141,7 @@ export class DesktopNarration {this.report({ phase: 'error', reason: 'missing-api-key' });return null;}- const mic = new MicRecorder(assetUrl('pcm-processor.js'), {+ const mic = new MicRecorder(assetUrl('mic-frames.js'), {onEpoch: () => undefined,onLevel: (level, speaking) => window.mimik.capture.narration({ level, speaking }),onStreamEnded: () => {diff --git a/apps/extension/public/mic-frames.js b/apps/extension/public/mic-frames.jsnew file mode 100644index 0000000..2dbf0a9--- /dev/null+++ b/apps/extension/public/mic-frames.js@@ -0,0 +1,36 @@+class MicFrameCollector extends AudioWorkletProcessor {+ constructor(options) {+ super();+ const frameMs = options?.processorOptions?.frameMs ?? 250;+ this.pending = new Float32Array(Math.max(1, Math.round((sampleRate * frameMs) / 1000)));+ this.filled = 0;+ }++ process(inputs) {+ const mono = inputs[0]?.[0];+ if (!mono) return true;+ let read = 0;+ while (read < mono.length) {+ const room = this.pending.length - this.filled;+ const take = Math.min(room, mono.length - read);+ this.pending.set(mono.subarray(read, read + take), this.filled);+ this.filled += take;+ read += take;+ if (this.filled === this.pending.length) this.flush();+ }+ return true;+ }++ flush() {+ const bytes = new ArrayBuffer(this.pending.length * 2);+ const view = new DataView(bytes);+ for (let i = 0; i < this.pending.length; i++) {+ const level = Math.max(-1, Math.min(1, this.pending[i]));+ view.setInt16(i * 2, Math.round(level * 0x7fff), true);+ }+ this.port.postMessage(bytes, [bytes]);+ this.filled = 0;+ }+}++registerProcessor('mic-frames', MicFrameCollector);diff --git a/apps/extension/public/pcm-processor.js b/apps/extension/public/pcm-processor.jsdeleted file mode 100644index 522a0e8..0000000--- a/apps/extension/public/pcm-processor.js+++ /dev/null@@ -1,25 +0,0 @@-const CHUNK = 4800;--class PCMProcessor extends AudioWorkletProcessor {- constructor() {- super();- this.buffer = new Int16Array(CHUNK);- this.index = 0;- }-- process(inputs) {- const channel = inputs[0] && inputs[0][0];- if (!channel) return true;- for (const sample of channel) {- this.buffer[this.index++] = sample > 1 ? 32767 : sample < -1 ? -32767 : (sample * 32767) | 0;- if (this.index >= CHUNK) {- this.port.postMessage(this.buffer.buffer, [this.buffer.buffer]);- this.buffer = new Int16Array(CHUNK);- this.index = 0;- }- }- return true;- }-}--registerProcessor('pcm-processor', PCMProcessor);diff --git a/apps/extension/src/lib/voice/__tests__/mic-frames.test.ts b/apps/extension/src/lib/voice/__tests__/mic-frames.test.tsnew file mode 100644index 0000000..177480a--- /dev/null+++ b/apps/extension/src/lib/voice/__tests__/mic-frames.test.ts@@ -0,0 +1,57 @@+import { beforeAll, describe, expect, it, vi } from 'vitest';++type Collector = {+ port: { postMessage: ReturnType<typeof vi.fn> };+ process(inputs: Float32Array[][]): boolean;+};+type CollectorClass = new (options?: { processorOptions?: { frameMs?: number } }) => Collector;++let Registered: CollectorClass;+let registeredName = '';++beforeAll(async () => {+ Object.assign(globalThis, {+ sampleRate: 16000,+ AudioWorkletProcessor: class {+ port = { postMessage: vi.fn() };+ },+ registerProcessor: (name: string, processor: CollectorClass) => {+ registeredName = name;+ Registered = processor;+ },+ });+ await import(new URL('../../../../public/mic-frames.js', import.meta.url).href);+});++function posted(collector: Collector): Int16Array[] {+ return collector.port.postMessage.mock.calls.map(([buffer]) => new Int16Array(buffer as ArrayBuffer));+}++describe('mic-frames worklet', () => {+ it('registers under the name the recorder asks for', () => {+ expect(registeredName).toBe('mic-frames');+ });++ it('sends one frame per frameMs of audio, sized from the sample rate', () => {+ const collector = new Registered({ processorOptions: { frameMs: 1 } });+ collector.process([[new Float32Array(40)]]);++ expect(posted(collector).map((frame) => frame.length)).toEqual([16, 16]);+ });++ it('clamps and scales samples symmetrically into 16-bit integers', () => {+ const collector = new Registered({ processorOptions: { frameMs: 0.375 } });+ collector.process([[Float32Array.from([2, 1, 0.5, 0, -0.5, -1])]]);++ expect([...posted(collector)[0]!]).toEqual([32767, 32767, 16384, 0, -16383, -32767]);+ });++ it('carries a frame across render quanta and ignores an empty input', () => {+ const collector = new Registered({ processorOptions: { frameMs: 0.25 } });+ collector.process([[Float32Array.from([0.1, 0.2, 0.3])]]);+ collector.process([[]]);+ collector.process([[Float32Array.from([0.4, 0.5])]]);++ expect(posted(collector).map((frame) => frame.length)).toEqual([4]);+ });+});diff --git a/apps/extension/src/lib/voice/create-voice-host.ts b/apps/extension/src/lib/voice/create-voice-host.tsindex a67e824..207ba2c 100644--- a/apps/extension/src/lib/voice/create-voice-host.ts+++ b/apps/extension/src/lib/voice/create-voice-host.ts@@ -46,7 +46,7 @@ export function createVoiceHost(): VoiceHost {let retained: VoiceRecording | null = null;const slicer = new NarrationSlicer();- const recorder = new MicRecorder(getExtensionURL('/pcm-processor.js'), {+ const recorder = new MicRecorder(getExtensionURL('/mic-frames.js'), {onEpoch: (audioEpochMs) =>emit(voiceMessage<VoiceEpochEvent>({diff --git a/packages/core/src/capture/voice/mic-recorder.ts b/packages/core/src/capture/voice/mic-recorder.tsindex c447752..8eb1fec 100644--- a/packages/core/src/capture/voice/mic-recorder.ts+++ b/packages/core/src/capture/voice/mic-recorder.ts@@ -3,7 +3,8 @@ import { SPEECH_RMS_THRESHOLD } from './energy-gate';import { openStream } from './open-stream';const TARGET_SAMPLE_RATE = 16000;-const PCM_WORKLET_NAME = 'pcm-processor';+const MIC_FRAMES_WORKLET = 'mic-frames';+const FRAME_MS = 250;const ANALYSER_FFT_SIZE = 2048;const LEVEL_FLOOR_DB = -60;@@ -83,7 +84,7 @@ export class MicRecorder {this.analyser = context.createAnalyser();this.analyser.fftSize = ANALYSER_FFT_SIZE;this.analyserFrame = new Float32Array(this.analyser.fftSize);- this.worklet = new AudioWorkletNode(context, PCM_WORKLET_NAME);+ this.worklet = new AudioWorkletNode(context, MIC_FRAMES_WORKLET, { processorOptions: { frameMs: FRAME_MS } });this.worklet.port.onmessage = (event: MessageEvent<ArrayBuffer>) => this.appendFrame(event.data);this.source.connect(this.analyser);