Skip to content

armgit / mimik

Public Imported from GitHub
0

🎬 A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots πŸ“–

feat(voice): add Silero voice-activity-detection wrapper

Luis Ortiz69f078fa059b
diff --git a/package.json b/package.json
index c69520b..fac78a6 100644
--- a/package.json
+++ b/package.json
@@ -28,6 +28,7 @@
"@ai-sdk/anthropic": "^3.0.64",
"@ai-sdk/openai": "^3.0.49",
"@fontsource/poppins": "^5.2.7",
+ "@ricky0123/vad-web": "^0.0.30",
"@tailwindcss/vite": "^4.2.2",
"@webext-core/messaging": "^2.3.0",
"@wxt-dev/module-react": "^1.2.2",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d209c2c..ff4ed9b 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -22,6 +22,9 @@ importers:
'@fontsource/poppins':
specifier: ^5.2.7
version: 5.2.7
+ '@ricky0123/vad-web':
+ specifier: ^0.0.30
+ version: 0.0.30
'@tailwindcss/vite':
specifier: ^4.2.2
@@ -624,6 +627,33 @@ packages:
resolution: {integrity: sha512-h104Kh26rR8tm+a3Qkc5S4VLYint3FE48as7+/5oCEcKR2idC/pF1G6AhIXKI+eHPJa/3J9i5z0Al47IeGHPkA==}
engines: {node: '>=12'}
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
+
+ '@protobufjs/[email protected]':
+ resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==}
+
'@radix-ui/[email protected]':
resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==}
@@ -1314,6 +1344,9 @@ packages:
'@radix-ui/[email protected]':
resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==}
+ '@ricky0123/[email protected]':
+ resolution: {integrity: sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==}
+
'@rolldown/[email protected]':
resolution: {integrity: sha512-pv1y2Fv0JybcykuiiD3qBOBdz6RteYojRFY1d+b95WVuzx211CRh+ytI/+9iVyWQ6koTh5dawe4S/yRfOFjgaA==}
engines: {node: ^20.19.0 || >=22.12.0}
@@ -2195,6 +2228,9 @@ packages:
engines: {node: '>=18'}
hasBin: true
+ resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==}
+
resolution: {integrity: sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==}
engines: {node: '>= 18'}
@@ -2271,6 +2307,9 @@ packages:
resolution: {integrity: sha512-+xGQY0YyAWCnqy7Cd++hc2JqMYzlm0dG30Jd0beaA64sROr8C4nt8Yc9V5Ro3avlSUDTN0ulqP/VBKi1/lLygw==}
+ resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}
+
resolution: {integrity: sha512-Tz23LR9T9jOGVZm2x1EPdXqwA37G/owYMxRwU0E4miurAtFsPMQ1d2Jc2okUaSjZqAFz2oEn3FLXC5a0a+siyA==}
engines: {node: '>=20.0.0'}
@@ -2638,6 +2677,9 @@ packages:
resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}
+
resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}
hasBin: true
@@ -2774,6 +2816,12 @@ packages:
resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-3KxL5wIVqa8Ex08jxSzncm9CMgw8CjOFyOQ7SxvG9o0cVLlhTNKXyIQuTbtX4tGPJEf73OER2xrjt4HJSBL4ow==}
+
+ resolution: {integrity: sha512-ogDLsqIozHZwifPuN37OproAo0byX6t43/bP8GzeZWBWD6MOGExswFAx3up4NS/vvWBOg2u2PXomDt3rMmdQSg==}
+
resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==}
engines: {node: '>=20'}
@@ -2847,6 +2895,9 @@ packages:
resolution: {integrity: sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==}
+ resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
+
resolution: {integrity: sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==}
engines: {node: '>=18'}
@@ -2889,6 +2940,10 @@ packages:
resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==}
+ resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==}
+ engines: {node: '>=12.0.0'}
+
resolution: {integrity: sha512-QMQbWL0FWgBfnkJ6w8HOJoIPaWLE7vTpewM4ae2vLs7SrD4eKdAk+SxOzqAICwbhEPuaLAOA+XkT9sZS5R0PmA==}
engines: {node: '>=18.0.0'}
@@ -4041,6 +4096,26 @@ snapshots:
'@pnpm/network.ca-file': 1.0.2
config-chain: 1.1.13
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]':
+ dependencies:
+ '@protobufjs/aspromise': 1.1.2
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
+ '@protobufjs/[email protected]': {}
+
'@radix-ui/[email protected]': {}
'@radix-ui/[email protected]': {}
@@ -4788,6 +4863,10 @@ snapshots:
'@radix-ui/[email protected]': {}
+ '@ricky0123/[email protected]':
+ dependencies:
+ onnxruntime-web: 1.27.0
+
'@rolldown/[email protected]':
optional: true
@@ -5594,6 +5673,8 @@ snapshots:
minimist: 1.2.8
xml2js: 0.6.2
+
@@ -5658,6 +5739,8 @@ snapshots:
+
dependencies:
'@types/node': 25.5.0
@@ -5999,6 +6082,8 @@ snapshots:
strip-ansi: 7.2.0
wrap-ansi: 9.0.2
+
dependencies:
js-tokens: 4.0.0
@@ -6127,6 +6212,17 @@ snapshots:
dependencies:
mimic-function: 5.0.1
+
+ dependencies:
+ flatbuffers: 25.9.23
+ guid-typescript: 1.0.9
+ long: 5.3.2
+ onnxruntime-common: 1.27.0
+ platform: 1.3.6
+ protobufjs: 7.6.5
+
dependencies:
default-browser: 5.5.0
@@ -6219,6 +6315,8 @@ snapshots:
exsolve: 1.0.8
pathe: 2.0.3
+
@@ -6262,6 +6360,20 @@ snapshots:
+ dependencies:
+ '@protobufjs/aspromise': 1.1.2
+ '@protobufjs/base64': 1.1.2
+ '@protobufjs/codegen': 2.0.5
+ '@protobufjs/eventemitter': 1.1.1
+ '@protobufjs/fetch': 1.1.1
+ '@protobufjs/float': 1.0.2
+ '@protobufjs/path': 1.1.2
+ '@protobufjs/pool': 1.1.0
+ '@protobufjs/utf8': 1.1.2
+ '@types/node': 25.5.0
+ long: 5.3.2
+
dependencies:
cac: 6.7.14
diff --git a/src/core/capture/voice/__tests__/vad.integration.test.ts b/src/core/capture/voice/__tests__/vad.integration.test.ts
new file mode 100644
index 0000000..3124368
--- /dev/null
+++ b/src/core/capture/voice/__tests__/vad.integration.test.ts
@@ -0,0 +1,61 @@
+import { readFile } from 'node:fs/promises';
+import { createRequire } from 'node:module';
+import { describe, expect, it } from 'vitest';
+import { createSpeechDetector } from '../vad';
+
+const require = createRequire(import.meta.url);
+
+const detectWithLocalModel = createSpeechDetector({
+ modelURL: require.resolve('@ricky0123/vad-web/dist/silero_vad_legacy.onnx'),
+ modelFetcher: async (path: string) => {
+ const file = await readFile(path);
+ return file.buffer.slice(file.byteOffset, file.byteOffset + file.byteLength) as ArrayBuffer;
+ },
+});
+
+const SAMPLE_RATE = 16000;
+const DURATION_S = 6;
+
+function synth(shape: (i: number) => number): Int16Array {
+ const pcm = new Int16Array(SAMPLE_RATE * DURATION_S);
+ for (let i = 0; i < pcm.length; i += 1) {
+ pcm[i] = Math.max(-32768, Math.min(32767, Math.round(shape(i))));
+ }
+ return pcm;
+}
+
+function pseudoRandom(seed: number): () => number {
+ let state = seed;
+ return () => {
+ state = (state * 1664525 + 1013904223) % 4294967296;
+ return state / 4294967296;
+ };
+}
+
+describe('detectSpeech against real Silero', () => {
+ const nonSpeech: Array<[string, Int16Array]> = [
+ ['digital silence', synth(() => 0)],
+ ['sine tone at 220Hz', synth((i) => 8000 * Math.sin((2 * Math.PI * 220 * i) / SAMPLE_RATE))],
+ ['mains hum at 60Hz', synth((i) => 12000 * Math.sin((2 * Math.PI * 60 * i) / SAMPLE_RATE))],
+ [
+ 'loud white noise',
+ (() => {
+ const rand = pseudoRandom(7);
+ return synth(() => (rand() * 2 - 1) * 20000);
+ })(),
+ ],
+ [
+ 'room-level white noise',
+ (() => {
+ const rand = pseudoRandom(11);
+ return synth(() => (rand() * 2 - 1) * 600);
+ })(),
+ ],
+ ];
+
+ for (const [name, pcm] of nonSpeech) {
+ it(`returns no segments for ${name}`, async () => {
+ await expect(detectWithLocalModel(pcm, SAMPLE_RATE)).resolves.toEqual([]);
+ }, 60000);
+ }
+});
diff --git a/src/core/capture/voice/__tests__/vad.test.ts b/src/core/capture/voice/__tests__/vad.test.ts
new file mode 100644
index 0000000..b0c2190
--- /dev/null
+++ b/src/core/capture/voice/__tests__/vad.test.ts
@@ -0,0 +1,137 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { AbsoluteSeconds } from '../types';
+import {
+ createSpeechDetector,
+ detectSpeech,
+ MIN_SPEECH_FRAMES,
+ MS_PER_FRAME,
+ PRE_SPEECH_PAD_FRAMES,
+ REDEMPTION_FRAMES,
+} from '../vad';
+
+const spy = vi.hoisted(() => ({
+ detected: [] as Array<{ start: number; end: number }>,
+ audio: null as Float32Array | null,
+ sampleRate: 0,
+ options: {} as Record<string, unknown>,
+}));
+
+vi.mock('@ricky0123/vad-web', () => ({
+ NonRealTimeVAD: {
+ new: async (options: Record<string, unknown>) => {
+ spy.options = options;
+ return {
+ async *run(audio: Float32Array, sampleRate: number) {
+ spy.audio = audio;
+ spy.sampleRate = sampleRate;
+ for (const item of spy.detected) yield { audio: new Float32Array(0), ...item };
+ },
+ };
+ },
+ },
+}));
+
+const givenDetected = (items: Array<{ start: number; end: number }>) => {
+ spy.detected = items;
+};
+
+describe('detectSpeech', () => {
+ it('converts Int16 samples to Float32 by dividing by 32768', async () => {
+ givenDetected([]);
+ await detectSpeech(new Int16Array([-32768, -1, 0, 1, 32767]), 16000);
+
+ expect(Array.from(spy.audio ?? [])).toEqual([-1, -1 / 32768, 0, 1 / 32768, 32767 / 32768]);
+ expect(spy.audio).toBeInstanceOf(Float32Array);
+ });
+
+ it('keeps every converted sample inside the normalised range', async () => {
+ givenDetected([]);
+ await detectSpeech(new Int16Array([-32768, 32767, -16384, 16384]), 16000);
+
+ const values = Array.from(spy.audio ?? []);
+ expect(values.every((v) => v >= -1 && v <= 1)).toBe(true);
+ expect(values[2]).toBe(-0.5);
+ expect(values[3]).toBe(0.5);
+ });
+
+ it('forwards the source sample rate untouched', async () => {
+ givenDetected([]);
+ await detectSpeech(new Int16Array(16), 48000);
+
+ expect(spy.sampleRate).toBe(48000);
+ });
+
+ it('converts library milliseconds into seconds', async () => {
+ givenDetected([
+ { start: 1500, end: 3250 },
+ { start: 4000, end: 4500 },
+ ]);
+ const segments = await detectSpeech(new Int16Array(16), 16000);
+
+ expect(segments).toEqual([
+ { start: 1.5, end: 3.25 },
+ { start: 4, end: 4.5 },
+ ]);
+ });
+
+ it('returns segments in ascending start order', async () => {
+ givenDetected([
+ { start: 9000, end: 9500 },
+ { start: 1000, end: 1500 },
+ { start: 5000, end: 5500 },
+ ]);
+ const segments = await detectSpeech(new Int16Array(16), 16000);
+
+ expect(segments.map((s) => s.start)).toEqual([1, 5, 9]);
+ expect(segments.map((s) => s.end)).toEqual([1.5, 5.5, 9.5]);
+ });
+
+ it('brands its output as AbsoluteSeconds', async () => {
+ givenDetected([{ start: 2000, end: 4000 }]);
+ const segments = await detectSpeech(new Int16Array(16), 16000);
+
+ const start: AbsoluteSeconds = segments[0].start;
+ const end: AbsoluteSeconds = segments[0].end;
+ expect(start).toBe(2);
+ expect(end).toBe(4);
+ });
+
+ it('returns an empty array when the VAD finds no speech', async () => {
+ givenDetected([]);
+ const segments = await detectSpeech(new Int16Array(16000 * 5), 16000);
+
+ expect(segments).toEqual([]);
+ });
+
+ it('passes the narration-tuned thresholds and frame counts to the VAD', async () => {
+ givenDetected([]);
+ await detectSpeech(new Int16Array(16), 16000);
+
+ expect(spy.options).toMatchObject({
+ positiveSpeechThreshold: 0.5,
+ negativeSpeechThreshold: 0.35,
+ minSpeechMs: MIN_SPEECH_FRAMES * MS_PER_FRAME,
+ redemptionMs: REDEMPTION_FRAMES * MS_PER_FRAME,
+ preSpeechPadMs: PRE_SPEECH_PAD_FRAMES * MS_PER_FRAME,
+ });
+ expect(MS_PER_FRAME).toBe(96);
+ expect(spy.options.minSpeechMs).toBe(384);
+ expect(spy.options.redemptionMs).toBe(1152);
+ expect(spy.options.preSpeechPadMs).toBe(480);
+ });
+
+ it('lets a caller override asset locations without changing the tuning', async () => {
+ givenDetected([]);
+ const ortConfig = vi.fn();
+ await createSpeechDetector({ modelURL: 'chrome-extension://abc/silero.onnx', ortConfig })(
+ new Int16Array(16),
+ 16000,
+ );
+
+ expect(spy.options).toMatchObject({
+ modelURL: 'chrome-extension://abc/silero.onnx',
+ ortConfig,
+ positiveSpeechThreshold: 0.5,
+ });
+ });
+});
diff --git a/src/core/capture/voice/vad.ts b/src/core/capture/voice/vad.ts
new file mode 100644
index 0000000..7777274
--- /dev/null
+++ b/src/core/capture/voice/vad.ts
@@ -0,0 +1,50 @@
+import { absoluteSeconds, type SpeechSegment } from './types';
+
+export const FRAME_SAMPLES = 1536;
+export const MS_PER_FRAME = FRAME_SAMPLES / 16;
+
+export const POSITIVE_SPEECH_THRESHOLD = 0.5;
+export const NEGATIVE_SPEECH_THRESHOLD = 0.35;
+export const MIN_SPEECH_FRAMES = 4;
+export const REDEMPTION_FRAMES = 12;
+export const PRE_SPEECH_PAD_FRAMES = 5;
+
+const INT16_SCALE = 32768;
+
+export interface VadAssetOptions {
+ modelURL?: string;
+ modelFetcher?: (path: string) => Promise<ArrayBuffer>;
+ ortConfig?: (ort: any) => void;
+}
+
+export function toFloat32(pcm: Int16Array): Float32Array {
+ const out = new Float32Array(pcm.length);
+ for (let i = 0; i < pcm.length; i += 1) out[i] = pcm[i] / INT16_SCALE;
+ return out;
+}
+
+export function createSpeechDetector(assets: VadAssetOptions = {}) {
+ return async function detectSpeech(pcm: Int16Array, sampleRate: number): Promise<SpeechSegment[]> {
+ const { NonRealTimeVAD } = await import('@ricky0123/vad-web');
+ const vad = await NonRealTimeVAD.new({
+ positiveSpeechThreshold: POSITIVE_SPEECH_THRESHOLD,
+ negativeSpeechThreshold: NEGATIVE_SPEECH_THRESHOLD,
+ minSpeechMs: MIN_SPEECH_FRAMES * MS_PER_FRAME,
+ redemptionMs: REDEMPTION_FRAMES * MS_PER_FRAME,
+ preSpeechPadMs: PRE_SPEECH_PAD_FRAMES * MS_PER_FRAME,
+ ...assets,
+ });
+
+ const segments: SpeechSegment[] = [];
+ for await (const detected of vad.run(toFloat32(pcm), sampleRate)) {
+ segments.push({
+ start: absoluteSeconds(detected.start / 1000),
+ end: absoluteSeconds(detected.end / 1000),
+ });
+ }
+
+ return segments.sort((a, b) => a.start - b.start);
+ };
+}
+
+export const detectSpeech = createSpeechDetector();