Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(desktop): stop meeting audio from stuttering under load
Elian Doran0f59f2e0b1ea
diff --git a/apps/desktop/src/services/transcription/capture.ts b/apps/desktop/src/services/transcription/capture.tsindex 8b7aaa2ce2..50a4100d5e 100644--- a/apps/desktop/src/services/transcription/capture.ts+++ b/apps/desktop/src/services/transcription/capture.ts@@ -21,6 +21,9 @@ export interface AudioCapture {const SAMPLES_CHANNEL = "transcription-samples";+/** Samples per IPC message. `VadSegmenter` drops what does not fill a 512-sample window. */+const CHUNK_SAMPLES = 4096;+/** The page function that starts each source. */const SOURCE_STARTERS = [[ "microphone", "captureMicrophone()" ],@@ -111,6 +114,7 @@ async function createCaptureWindow() {contextIsolation: false,sandbox: false,autoplayPolicy: "no-user-gesture-required",+ backgroundThrottling: false,// Without hardware acceleration, a hidden window renders no frames, so the// self-capture in CAPTURE_PAGE times out waiting for its video source.offscreen: true@@ -122,17 +126,58 @@ async function createCaptureWindow() {return window;}+/**+ * The audio worklet that collects a source into chunks of `processorOptions.chunkSamples`. It runs+ * on the audio rendering thread, where a busy page cannot delay it and corrupt the start of a+ * chunk. `.claude/skills/working-with-transcription/capture-test.mjs` reads it from this file.+ */+const CAPTURE_PROCESSOR = `+class ChunkCollector extends AudioWorkletProcessor {+ constructor({ processorOptions }) {+ super();+ this.chunkSamples = processorOptions.chunkSamples;+ this.chunk = new Float32Array(this.chunkSamples);+ this.filled = 0;+ }++ process(inputs) {+ const input = inputs[0][0];+ let offset = 0;+ while (input && offset < input.length) {+ const count = Math.min(input.length - offset, this.chunkSamples - this.filled);+ this.chunk.set(input.subarray(offset, offset + count), this.filled);+ this.filled += count;+ offset += count;+ if (this.filled === this.chunkSamples) {+ this.port.postMessage(this.chunk, [ this.chunk.buffer ]);+ this.chunk = new Float32Array(this.chunkSamples);+ this.filled = 0;+ }+ }+ return true;+ }+}+registerProcessor("chunk-collector", ChunkCollector);+`;+const CAPTURE_PAGE = `<!DOCTYPE html><script>const { ipcRenderer } = require("electron");const context = new AudioContext({ sampleRate: ${SAMPLE_RATE} });--function pipe(source, media) {- const processor = context.createScriptProcessor(4096, 1, 1);- processor.onaudioprocess = (e) =>- ipcRenderer.send("${SAMPLES_CHANNEL}", source, e.inputBuffer.getChannelData(0).slice());- context.createMediaStreamSource(media).connect(processor);- processor.connect(context.destination);+const processorLoaded = context.audioWorklet.addModule(URL.createObjectURL(+ new Blob([ ${JSON.stringify(CAPTURE_PROCESSOR)} ], { type: "text/javascript" })));++async function pipe(source, media) {+ await processorLoaded;+ const collector = new AudioWorkletNode(context, "chunk-collector", {+ numberOfInputs: 1,+ numberOfOutputs: 0,+ channelCount: 1,+ channelCountMode: "explicit",+ processorOptions: { chunkSamples: ${CHUNK_SAMPLES} }+ });+ collector.port.onmessage = (e) => ipcRenderer.send("${SAMPLES_CHANNEL}", source, e.data);+ context.createMediaStreamSource(media).connect(collector);}function describe(tracks) {@@ -143,7 +188,7 @@ async function captureMicrophone() {const media = await navigator.mediaDevices.getUserMedia({audio: { channelCount: 1, echoCancellation: true, noiseSuppression: false }});- pipe("microphone", media);+ await pipe("microphone", media);return describe(media.getAudioTracks());}@@ -165,7 +210,7 @@ async function captureSystemAudio() {console.log("System track muted=" + track.muted + " settings="+ JSON.stringify(track.getSettings()));}- pipe("system", new MediaStream(audioTracks));+ await pipe("system", new MediaStream(audioTracks));return describe(audioTracks);}</script>`;