Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
chore(standalone): transcribe a shared tab's audio in the WebAssembly prototype
Elian Doran4bad42ff6785
diff --git a/apps/standalone/transcription-poc/README.md b/apps/standalone/transcription-poc/README.mdindex 391d29739a..96e0c6d5b4 100644--- a/apps/standalone/transcription-poc/README.md+++ b/apps/standalone/transcription-poc/README.md@@ -9,8 +9,13 @@ node apps/standalone/transcription-poc/fetch-assets.mjs # once: the wasm buildnode apps/standalone/transcription-poc/serve.mjs # http://localhost:5555/```-Pick a model, *Download to OPFS* (from Hugging Face, pinned revisions), *Load*, then *Transcribe clip* or-*Listen to the microphone*. Timings and the WebAssembly memory size go to the log.+Pick a model, *Download to OPFS* (from Hugging Face, pinned revisions), *Load*, then *Transcribe clip*,+*Listen to the microphone* or *Listen to a tab*; the two live sources can run together, each with its own+VAD. Timings and the WebAssembly memory size go to the log.++*Listen to a tab* uses `getDisplayMedia`, so the browser asks which tab or screen to share, and audio+arrives only when "Share tab audio" (or "Share system audio") is ticked. Firefox and Safari return no+audio from it.## How it worksdiff --git a/apps/standalone/transcription-poc/index.html b/apps/standalone/transcription-poc/index.htmlindex 2efd497fce..39aecf7daa 100644--- a/apps/standalone/transcription-poc/index.html+++ b/apps/standalone/transcription-poc/index.html@@ -27,7 +27,8 @@<legend>Transcribe</legend><select id="clip"></select><button id="transcribe">Transcribe clip</button>- <button id="listen">Listen to the microphone</button>+ <button id="listen-microphone"></button>+ <button id="listen-tab"></button></fieldset><pre id="log"></pre>diff --git a/apps/standalone/transcription-poc/main.js b/apps/standalone/transcription-poc/main.jsindex f2a6001d33..354ab08fda 100644--- a/apps/standalone/transcription-poc/main.js+++ b/apps/standalone/transcription-poc/main.js@@ -69,9 +69,32 @@ const CLIPS = [[ "German, 4 s", `${HF}/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3/resolve/1a468a35cbba69418f126de829e75261dea4a4e4/test_wavs/de.wav` ]];+/** The live sources. Each runs its own VAD in the worker and is labeled in the log. */+const SOURCES = {+ microphone: {+ button: "listen-microphone",+ label: "Listen to the microphone",+ // Echo cancellation keeps the tab's audio, played through the speakers, out of the microphone.+ open: () => navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true } })+ },+ tab: {+ button: "listen-tab",+ label: "Listen to a tab",+ // Chrome requires video to share audio; the video track is never read.+ open: () => navigator.mediaDevices.getDisplayMedia({+ video: true,+ audio: { echoCancellation: false, noiseSuppression: false, autoGainControl: false },+ systemAudio: "include",+ selfBrowserSurface: "exclude",+ preferCurrentTab: false+ })+ }+};+const $ = (id) => document.getElementById(id);const worker = new Worker("worker.js");-let microphone = null;+/** The running captures, by source. */+const captures = new Map();for (const model of MODELS) {$("model").append(new Option(model.label, model.id));@@ -87,9 +110,10 @@ worker.onmessage = ({ data }) => {case "downloaded": return void reportPersistence();case "loaded": return log(`Loaded ${data.model}: copy into MEMFS ${ms(data.timings.copyToMemfsMs)}, create recognizer ${ms(data.timings.createRecognizerMs)}, WebAssembly memory ${data.heapAfterLoad} MiB.`);case "result": return log(`${data.label}: ${ms(data.ms)} for ${ms(data.audioMs)} of audio (RTF ${data.rtf.toFixed(2)}), memory ${data.heap} MiB\n → ${data.text}`);- case "utterance": return log(`[${ms(data.offsetMs)}] ${data.text}\n (${ms(data.ms)} for ${ms(data.audioMs)}, RTF ${data.rtf.toFixed(2)}, lag ${ms(data.lagMs)})`);- case "live-started": return log("Listening.");- case "live-stopped": return log("Stopped listening.");+ case "utterance": return log(`${data.source} [${ms(data.offsetMs)}] ${data.text}\n (${ms(data.ms)} for ${ms(data.audioMs)}, RTF ${data.rtf.toFixed(2)}, lag ${ms(data.lagMs)})`);+ case "peak": return log(`${data.source}: loudest sample ${data.peak.toFixed(4)} in the last 5 s`);+ case "live-started": return log(`${data.source}: listening.`);+ case "live-stopped": return log(`${data.source}: stopped listening.`);case "deleted": return log(`Deleted ${data.model} from OPFS.`);case "log": return console.debug(data.text);case "error": return log(`Error: ${data.message}`);@@ -108,19 +132,39 @@ $("transcribe").onclick = async () => {const samples = await decodeClip(url);worker.postMessage({ type: "transcribe", label, samples }, [ samples.buffer ]);};-$("listen").onclick = async () => {- if (microphone) {- microphone.stop();- microphone = null;- worker.postMessage({ type: "live-stop" });- $("listen").textContent = "Listen to the microphone";- return;+for (const [ source, { button, label, open } ] of Object.entries(SOURCES)) {+ $(button).textContent = label;+ $(button).onclick = () => (captures.has(source) ? stopSource(source) : startSource(source, open));+}++async function startSource(source, open) {+ let media;+ try {+ media = await open();+ } catch (e) {+ return log(`${source}: ${e.message}`);}- worker.postMessage({ type: "live-start" });- microphone = await captureMicrophone((samples) =>- worker.postMessage({ type: "samples", samples }, [ samples.buffer ]));- $("listen").textContent = "Stop listening";-};+ const [ track ] = media.getAudioTracks();+ if (!track) {+ stopTracks(media);+ return log(`${source}: nothing to listen to. Share a tab and tick "Share tab audio" (or "Share system audio" for a whole screen).`);+ }+ log(`${source}: ${track.label}, ${JSON.stringify(track.getSettings())}`);+ // Ending the share from the browser's own bar ends the track.+ track.addEventListener("ended", () => stopSource(source));++ worker.postMessage({ type: "live-start", source });+ captures.set(source, captureAudio(media, (samples) =>+ worker.postMessage({ type: "samples", source, samples }, [ samples.buffer ])));+ $(SOURCES[source].button).textContent = `Stop: ${SOURCES[source].label.toLowerCase()}`;+}++function stopSource(source) {+ captures.get(source)?.stop();+ captures.delete(source);+ worker.postMessage({ type: "live-stop", source });+ $(SOURCES[source].button).textContent = SOURCES[source].label;+}/** Asks the browser not to evict the models under storage pressure. Only the page can ask. */async function reportPersistence() {@@ -149,23 +193,28 @@ async function decodeClip(url) {}}-async function captureMicrophone(onSamples) {- const media = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } });+/** Feeds a stream's audio, downmixed and resampled to mono 16 kHz, to `onSamples`. */+function captureAudio(media, onSamples) {const context = new AudioContext({ sampleRate: 16000 });const processor = context.createScriptProcessor(4096, 1, 1);processor.onaudioprocess = (e) => onSamples(e.inputBuffer.getChannelData(0).slice());- context.createMediaStreamSource(media).connect(processor);+ // Only the audio tracks: a MediaStreamSource rejects a stream whose first track is video.+ context.createMediaStreamSource(new MediaStream(media.getAudioTracks())).connect(processor);processor.connect(context.destination);return {stop() {- for (const track of media.getTracks()) {- track.stop();- }+ stopTracks(media);void context.close();}};}+function stopTracks(media) {+ for (const track of media.getTracks()) {+ track.stop();+ }+}+function showProgress({ file, loaded, total, cached, ms: took }) {const percent = Math.round(loaded / total * 100);$("progress").textContent = cacheddiff --git a/apps/standalone/transcription-poc/worker.js b/apps/standalone/transcription-poc/worker.jsindex 86854944ec..fbecf38240 100644--- a/apps/standalone/transcription-poc/worker.js+++ b/apps/standalone/transcription-poc/worker.js@@ -18,9 +18,8 @@ self.Module = {importScripts("vendor/sherpa-onnx-wasm-main-vad-asr.js", "vendor/sherpa-onnx-asr.js", "vendor/sherpa-onnx-vad.js");let recognizer = null;-let vad = null;-let pending = new Float32Array(0);-let liveSamples = 0;+/** One VAD per live source ("microphone", "tab"), so their utterances never mix. */+const lives = new Map();self.onmessage = async ({ data }) => {try {@@ -28,9 +27,9 @@ self.onmessage = async ({ data }) => {case "download": return await download(data.model);case "load": return await load(data.model);case "transcribe": return transcribeClip(data.label, data.samples);- case "live-start": return startLive();- case "samples": return acceptLive(data.samples);- case "live-stop": return stopLive();+ case "live-start": return startLive(data.source);+ case "samples": return acceptLive(data.source, data.samples);+ case "live-stop": return stopLive(data.source);case "delete": return await deleteModel(data.model);}} catch (e) {@@ -127,10 +126,11 @@ async function load(model) {}function unload() {+ for (const source of lives.keys()) {+ stopLive(source);+ }recognizer?.free();recognizer = null;- vad?.free();- vad = null;}function transcribeClip(label, samples) {@@ -142,10 +142,10 @@ function transcribeClip(label, samples) {post("result", { label, text, ms, audioMs, rtf: ms / audioMs, heap: heapMiB() });}-function startLive() {+function startLive(source) {requireRecognizer();- vad?.free();- vad = createVad(Module, {+ lives.get(source)?.vad.free();+ const vad = createVad(Module, {sileroVad: {model: "/models/silero_vad.onnx",threshold: 0.5,@@ -161,49 +161,63 @@ function startLive() {debug: 0,bufferSizeInSeconds: 60});- pending = new Float32Array(0);- liveSamples = 0;- post("live-started", {});+ lives.set(source, { vad, pending: new Float32Array(0), samples: 0, peak: 0, peakSince: performance.now() });+ post("live-started", { source });}-function acceptLive(samples) {- if (!vad) {+function acceptLive(source, samples) {+ const live = lives.get(source);+ if (!live) {return;}- liveSamples += samples.length;- const all = new Float32Array(pending.length + samples.length);- all.set(pending);- all.set(samples, pending.length);+ live.samples += samples.length;+ reportPeak(source, live, samples);+ const all = new Float32Array(live.pending.length + samples.length);+ all.set(live.pending);+ all.set(samples, live.pending.length);let offset = 0;for (; offset + VAD_WINDOW <= all.length; offset += VAD_WINDOW) {- vad.acceptWaveform(all.subarray(offset, offset + VAD_WINDOW));+ live.vad.acceptWaveform(all.subarray(offset, offset + VAD_WINDOW));}- pending = all.slice(offset);- drainVad();+ live.pending = all.slice(offset);+ drainVad(source, live);}-function stopLive() {- if (!vad) {+function stopLive(source) {+ const live = lives.get(source);+ if (!live) {return;}- vad.flush();- drainVad();- vad.free();- vad = null;- post("live-stopped", {});+ live.vad.flush();+ drainVad(source, live);+ live.vad.free();+ lives.delete(source);+ post("live-stopped", { source });+}++/** Logs the loudest sample every 5 s, to tell a silent capture from one without speech. */+function reportPeak(source, live, samples) {+ for (const sample of samples) {+ live.peak = Math.max(live.peak, Math.abs(sample));+ }+ if (performance.now() - live.peakSince >= 5000) {+ post("peak", { source, peak: live.peak });+ live.peak = 0;+ live.peakSince = performance.now();+ }}-function drainVad() {- while (!vad.isEmpty()) {- const segment = vad.front();- vad.pop();+function drainVad(source, live) {+ while (!live.vad.isEmpty()) {+ const segment = live.vad.front();+ live.vad.pop();const started = performance.now();const text = decode(segment.samples);const ms = performance.now() - started;const audioMs = segment.samples.length / SAMPLE_RATE * 1000;// How far the transcript trails the speaker once this utterance is out.- const lagMs = (liveSamples - segment.start) / SAMPLE_RATE * 1000 - audioMs + ms;- post("utterance", { text, ms, audioMs, rtf: ms / audioMs, lagMs, offsetMs: segment.start / SAMPLE_RATE * 1000 });+ const lagMs = (live.samples - segment.start) / SAMPLE_RATE * 1000 - audioMs + ms;+ post("utterance", { source, text, ms, audioMs, rtf: ms / audioMs, lagMs, offsetMs: segment.start / SAMPLE_RATE * 1000 });}}