π¬ A browser extension and desktop app that turns your clicks into step-by-step guides with annotated screenshots π
TypeScript browser-extensionchrome-extensiondocumentationfirefox-extensionguide-generatorlocal-firstmanifest-v3privacy-firstproductivityreactscreenshotstep-by-step-guidetutorialtypescriptworkflowwxt
feat(voice): add Silero voice-activity-detection wrapper
Luis Ortiz69f078fa059b
diff --git a/package.json b/package.jsonindex c69520b..fac78a6 100644--- a/package.json+++ b/package.json@@ -28,6 +28,7 @@"@ai-sdk/anthropic": "^3.0.64","@ai-sdk/openai": "^3.0.49","@fontsource/poppins": "^5.2.7",+ "@ricky0123/vad-web": "^0.0.30","@tailwindcss/vite": "^4.2.2","@webext-core/messaging": "^2.3.0","@wxt-dev/module-react": "^1.2.2",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex d209c2c..ff4ed9b 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -22,6 +22,9 @@ importers:'@fontsource/poppins':specifier: ^5.2.7version: 5.2.7+ '@ricky0123/vad-web':+ specifier: ^0.0.30+ version: 0.0.30'@tailwindcss/vite':specifier: ^4.2.2@@ -624,6 +627,33 @@ packages:resolution: {integrity: sha512-h104Kh26rR8tm+a3Qkc5S4VLYint3FE48as7+/5oCEcKR2idC/pF1G6AhIXKI+eHPJa/3J9i5z0Al47IeGHPkA==}engines: {node: '>=12'}+ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}++ '@protobufjs/[email protected]':+ resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==}+'@radix-ui/[email protected]':resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==}@@ -1314,6 +1344,9 @@ packages:'@radix-ui/[email protected]':resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==}+ '@ricky0123/[email protected]':+ resolution: {integrity: sha512-cJyYrh4YeeUBJcbR9Bic/bFDyB9qBkAepvpuWM3vLxnAi7bC3VHzf51UeNdT+OtY4D7MLAgV8iJMc4z41ZnaWg==}+'@rolldown/[email protected]':resolution: {integrity: sha512-pv1y2Fv0JybcykuiiD3qBOBdz6RteYojRFY1d+b95WVuzx211CRh+ytI/+9iVyWQ6koTh5dawe4S/yRfOFjgaA==}engines: {node: ^20.19.0 || >=22.12.0}@@ -2195,6 +2228,9 @@ packages:engines: {node: '>=18'}hasBin: true+ resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==}+resolution: {integrity: sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==}engines: {node: '>= 18'}@@ -2271,6 +2307,9 @@ packages:resolution: {integrity: sha512-+xGQY0YyAWCnqy7Cd++hc2JqMYzlm0dG30Jd0beaA64sROr8C4nt8Yc9V5Ro3avlSUDTN0ulqP/VBKi1/lLygw==}+ resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}+resolution: {integrity: sha512-Tz23LR9T9jOGVZm2x1EPdXqwA37G/owYMxRwU0E4miurAtFsPMQ1d2Jc2okUaSjZqAFz2oEn3FLXC5a0a+siyA==}engines: {node: '>=20.0.0'}@@ -2638,6 +2677,9 @@ packages:resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==}engines: {node: '>=18'}+ resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}+resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==}hasBin: true@@ -2774,6 +2816,12 @@ packages:resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==}engines: {node: '>=18'}+ resolution: {integrity: sha512-3KxL5wIVqa8Ex08jxSzncm9CMgw8CjOFyOQ7SxvG9o0cVLlhTNKXyIQuTbtX4tGPJEf73OER2xrjt4HJSBL4ow==}++ resolution: {integrity: sha512-ogDLsqIozHZwifPuN37OproAo0byX6t43/bP8GzeZWBWD6MOGExswFAx3up4NS/vvWBOg2u2PXomDt3rMmdQSg==}+resolution: {integrity: sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==}engines: {node: '>=20'}@@ -2847,6 +2895,9 @@ packages:resolution: {integrity: sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==}+ resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}+resolution: {integrity: sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==}engines: {node: '>=18'}@@ -2889,6 +2940,10 @@ packages:resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==}+ resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==}+ engines: {node: '>=12.0.0'}+resolution: {integrity: sha512-QMQbWL0FWgBfnkJ6w8HOJoIPaWLE7vTpewM4ae2vLs7SrD4eKdAk+SxOzqAICwbhEPuaLAOA+XkT9sZS5R0PmA==}engines: {node: '>=18.0.0'}@@ -4041,6 +4096,26 @@ snapshots:'@pnpm/network.ca-file': 1.0.2config-chain: 1.1.13+ '@protobufjs/[email protected]': {}++ '@protobufjs/[email protected]': {}++ '@protobufjs/[email protected]': {}++ '@protobufjs/[email protected]': {}++ '@protobufjs/[email protected]':+ dependencies:+ '@protobufjs/aspromise': 1.1.2++ '@protobufjs/[email protected]': {}++ '@protobufjs/[email protected]': {}++ '@protobufjs/[email protected]': {}++ '@protobufjs/[email protected]': {}+'@radix-ui/[email protected]': {}'@radix-ui/[email protected]': {}@@ -4788,6 +4863,10 @@ snapshots:'@radix-ui/[email protected]': {}+ '@ricky0123/[email protected]':+ dependencies:+ onnxruntime-web: 1.27.0+'@rolldown/[email protected]':optional: true@@ -5594,6 +5673,8 @@ snapshots:minimist: 1.2.8xml2js: 0.6.2+ [email protected]: {}+@@ -5658,6 +5739,8 @@ snapshots:+ [email protected]: {}+dependencies:'@types/node': 25.5.0@@ -5999,6 +6082,8 @@ snapshots:strip-ansi: 7.2.0wrap-ansi: 9.0.2+ [email protected]: {}+dependencies:js-tokens: 4.0.0@@ -6127,6 +6212,17 @@ snapshots:dependencies:mimic-function: 5.0.1+ [email protected]: {}++ dependencies:+ flatbuffers: 25.9.23+ guid-typescript: 1.0.9+ long: 5.3.2+ onnxruntime-common: 1.27.0+ platform: 1.3.6+ protobufjs: 7.6.5+dependencies:default-browser: 5.5.0@@ -6219,6 +6315,8 @@ snapshots:exsolve: 1.0.8pathe: 2.0.3+ [email protected]: {}+@@ -6262,6 +6360,20 @@ snapshots:+ dependencies:+ '@protobufjs/aspromise': 1.1.2+ '@protobufjs/base64': 1.1.2+ '@protobufjs/codegen': 2.0.5+ '@protobufjs/eventemitter': 1.1.1+ '@protobufjs/fetch': 1.1.1+ '@protobufjs/float': 1.0.2+ '@protobufjs/path': 1.1.2+ '@protobufjs/pool': 1.1.0+ '@protobufjs/utf8': 1.1.2+ '@types/node': 25.5.0+ long: 5.3.2+dependencies:cac: 6.7.14diff --git a/src/core/capture/voice/__tests__/vad.integration.test.ts b/src/core/capture/voice/__tests__/vad.integration.test.tsnew file mode 100644index 0000000..3124368--- /dev/null+++ b/src/core/capture/voice/__tests__/vad.integration.test.ts@@ -0,0 +1,61 @@+import { readFile } from 'node:fs/promises';+import { createRequire } from 'node:module';+import { describe, expect, it } from 'vitest';+import { createSpeechDetector } from '../vad';++const require = createRequire(import.meta.url);++const detectWithLocalModel = createSpeechDetector({+ modelURL: require.resolve('@ricky0123/vad-web/dist/silero_vad_legacy.onnx'),+ modelFetcher: async (path: string) => {+ const file = await readFile(path);+ return file.buffer.slice(file.byteOffset, file.byteOffset + file.byteLength) as ArrayBuffer;+ },+});++const SAMPLE_RATE = 16000;+const DURATION_S = 6;++function synth(shape: (i: number) => number): Int16Array {+ const pcm = new Int16Array(SAMPLE_RATE * DURATION_S);+ for (let i = 0; i < pcm.length; i += 1) {+ pcm[i] = Math.max(-32768, Math.min(32767, Math.round(shape(i))));+ }+ return pcm;+}++function pseudoRandom(seed: number): () => number {+ let state = seed;+ return () => {+ state = (state * 1664525 + 1013904223) % 4294967296;+ return state / 4294967296;+ };+}++describe('detectSpeech against real Silero', () => {+ const nonSpeech: Array<[string, Int16Array]> = [+ ['digital silence', synth(() => 0)],+ ['sine tone at 220Hz', synth((i) => 8000 * Math.sin((2 * Math.PI * 220 * i) / SAMPLE_RATE))],+ ['mains hum at 60Hz', synth((i) => 12000 * Math.sin((2 * Math.PI * 60 * i) / SAMPLE_RATE))],+ [+ 'loud white noise',+ (() => {+ const rand = pseudoRandom(7);+ return synth(() => (rand() * 2 - 1) * 20000);+ })(),+ ],+ [+ 'room-level white noise',+ (() => {+ const rand = pseudoRandom(11);+ return synth(() => (rand() * 2 - 1) * 600);+ })(),+ ],+ ];++ for (const [name, pcm] of nonSpeech) {+ it(`returns no segments for ${name}`, async () => {+ await expect(detectWithLocalModel(pcm, SAMPLE_RATE)).resolves.toEqual([]);+ }, 60000);+ }+});diff --git a/src/core/capture/voice/__tests__/vad.test.ts b/src/core/capture/voice/__tests__/vad.test.tsnew file mode 100644index 0000000..b0c2190--- /dev/null+++ b/src/core/capture/voice/__tests__/vad.test.ts@@ -0,0 +1,137 @@+import { describe, expect, it, vi } from 'vitest';+import type { AbsoluteSeconds } from '../types';+import {+ createSpeechDetector,+ detectSpeech,+ MIN_SPEECH_FRAMES,+ MS_PER_FRAME,+ PRE_SPEECH_PAD_FRAMES,+ REDEMPTION_FRAMES,+} from '../vad';++const spy = vi.hoisted(() => ({+ detected: [] as Array<{ start: number; end: number }>,+ audio: null as Float32Array | null,+ sampleRate: 0,+ options: {} as Record<string, unknown>,+}));++vi.mock('@ricky0123/vad-web', () => ({+ NonRealTimeVAD: {+ new: async (options: Record<string, unknown>) => {+ spy.options = options;+ return {+ async *run(audio: Float32Array, sampleRate: number) {+ spy.audio = audio;+ spy.sampleRate = sampleRate;+ for (const item of spy.detected) yield { audio: new Float32Array(0), ...item };+ },+ };+ },+ },+}));++const givenDetected = (items: Array<{ start: number; end: number }>) => {+ spy.detected = items;+};++describe('detectSpeech', () => {+ it('converts Int16 samples to Float32 by dividing by 32768', async () => {+ givenDetected([]);+ await detectSpeech(new Int16Array([-32768, -1, 0, 1, 32767]), 16000);++ expect(Array.from(spy.audio ?? [])).toEqual([-1, -1 / 32768, 0, 1 / 32768, 32767 / 32768]);+ expect(spy.audio).toBeInstanceOf(Float32Array);+ });++ it('keeps every converted sample inside the normalised range', async () => {+ givenDetected([]);+ await detectSpeech(new Int16Array([-32768, 32767, -16384, 16384]), 16000);++ const values = Array.from(spy.audio ?? []);+ expect(values.every((v) => v >= -1 && v <= 1)).toBe(true);+ expect(values[2]).toBe(-0.5);+ expect(values[3]).toBe(0.5);+ });++ it('forwards the source sample rate untouched', async () => {+ givenDetected([]);+ await detectSpeech(new Int16Array(16), 48000);++ expect(spy.sampleRate).toBe(48000);+ });++ it('converts library milliseconds into seconds', async () => {+ givenDetected([+ { start: 1500, end: 3250 },+ { start: 4000, end: 4500 },+ ]);+ const segments = await detectSpeech(new Int16Array(16), 16000);++ expect(segments).toEqual([+ { start: 1.5, end: 3.25 },+ { start: 4, end: 4.5 },+ ]);+ });++ it('returns segments in ascending start order', async () => {+ givenDetected([+ { start: 9000, end: 9500 },+ { start: 1000, end: 1500 },+ { start: 5000, end: 5500 },+ ]);+ const segments = await detectSpeech(new Int16Array(16), 16000);++ expect(segments.map((s) => s.start)).toEqual([1, 5, 9]);+ expect(segments.map((s) => s.end)).toEqual([1.5, 5.5, 9.5]);+ });++ it('brands its output as AbsoluteSeconds', async () => {+ givenDetected([{ start: 2000, end: 4000 }]);+ const segments = await detectSpeech(new Int16Array(16), 16000);++ const start: AbsoluteSeconds = segments[0].start;+ const end: AbsoluteSeconds = segments[0].end;+ expect(start).toBe(2);+ expect(end).toBe(4);+ });++ it('returns an empty array when the VAD finds no speech', async () => {+ givenDetected([]);+ const segments = await detectSpeech(new Int16Array(16000 * 5), 16000);++ expect(segments).toEqual([]);+ });++ it('passes the narration-tuned thresholds and frame counts to the VAD', async () => {+ givenDetected([]);+ await detectSpeech(new Int16Array(16), 16000);++ expect(spy.options).toMatchObject({+ positiveSpeechThreshold: 0.5,+ negativeSpeechThreshold: 0.35,+ minSpeechMs: MIN_SPEECH_FRAMES * MS_PER_FRAME,+ redemptionMs: REDEMPTION_FRAMES * MS_PER_FRAME,+ preSpeechPadMs: PRE_SPEECH_PAD_FRAMES * MS_PER_FRAME,+ });+ expect(MS_PER_FRAME).toBe(96);+ expect(spy.options.minSpeechMs).toBe(384);+ expect(spy.options.redemptionMs).toBe(1152);+ expect(spy.options.preSpeechPadMs).toBe(480);+ });++ it('lets a caller override asset locations without changing the tuning', async () => {+ givenDetected([]);+ const ortConfig = vi.fn();+ await createSpeechDetector({ modelURL: 'chrome-extension://abc/silero.onnx', ortConfig })(+ new Int16Array(16),+ 16000,+ );++ expect(spy.options).toMatchObject({+ modelURL: 'chrome-extension://abc/silero.onnx',+ ortConfig,+ positiveSpeechThreshold: 0.5,+ });+ });+});diff --git a/src/core/capture/voice/vad.ts b/src/core/capture/voice/vad.tsnew file mode 100644index 0000000..7777274--- /dev/null+++ b/src/core/capture/voice/vad.ts@@ -0,0 +1,50 @@+import { absoluteSeconds, type SpeechSegment } from './types';++export const FRAME_SAMPLES = 1536;+export const MS_PER_FRAME = FRAME_SAMPLES / 16;++export const POSITIVE_SPEECH_THRESHOLD = 0.5;+export const NEGATIVE_SPEECH_THRESHOLD = 0.35;+export const MIN_SPEECH_FRAMES = 4;+export const REDEMPTION_FRAMES = 12;+export const PRE_SPEECH_PAD_FRAMES = 5;++const INT16_SCALE = 32768;++export interface VadAssetOptions {+ modelURL?: string;+ modelFetcher?: (path: string) => Promise<ArrayBuffer>;+ ortConfig?: (ort: any) => void;+}++export function toFloat32(pcm: Int16Array): Float32Array {+ const out = new Float32Array(pcm.length);+ for (let i = 0; i < pcm.length; i += 1) out[i] = pcm[i] / INT16_SCALE;+ return out;+}++export function createSpeechDetector(assets: VadAssetOptions = {}) {+ return async function detectSpeech(pcm: Int16Array, sampleRate: number): Promise<SpeechSegment[]> {+ const { NonRealTimeVAD } = await import('@ricky0123/vad-web');+ const vad = await NonRealTimeVAD.new({+ positiveSpeechThreshold: POSITIVE_SPEECH_THRESHOLD,+ negativeSpeechThreshold: NEGATIVE_SPEECH_THRESHOLD,+ minSpeechMs: MIN_SPEECH_FRAMES * MS_PER_FRAME,+ redemptionMs: REDEMPTION_FRAMES * MS_PER_FRAME,+ preSpeechPadMs: PRE_SPEECH_PAD_FRAMES * MS_PER_FRAME,+ ...assets,+ });++ const segments: SpeechSegment[] = [];+ for await (const detected of vad.run(toFloat32(pcm), sampleRate)) {+ segments.push({+ start: absoluteSeconds(detected.start / 1000),+ end: absoluteSeconds(detected.end / 1000),+ });+ }++ return segments.sort((a, b) => a.start - b.start);+ };+}++export const detectSpeech = createSpeechDetector();