Skip to content

Build your personal knowledge base with Trilium Notes

chore(standalone): transcribe a shared tab's audio in the WebAssembly prototype

Elian Doran4bad42ff6785
diff --git a/apps/standalone/transcription-poc/README.md b/apps/standalone/transcription-poc/README.md
index 391d29739a..96e0c6d5b4 100644
--- a/apps/standalone/transcription-poc/README.md
+++ b/apps/standalone/transcription-poc/README.md
@@ -9,8 +9,13 @@ node apps/standalone/transcription-poc/fetch-assets.mjs # once: the wasm build
node apps/standalone/transcription-poc/serve.mjs # http://localhost:5555/
```
-Pick a model, *Download to OPFS* (from Hugging Face, pinned revisions), *Load*, then *Transcribe clip* or
-*Listen to the microphone*. Timings and the WebAssembly memory size go to the log.
+Pick a model, *Download to OPFS* (from Hugging Face, pinned revisions), *Load*, then *Transcribe clip*,
+*Listen to the microphone* or *Listen to a tab*; the two live sources can run together, each with its own
+VAD. Timings and the WebAssembly memory size go to the log.
+
+*Listen to a tab* uses `getDisplayMedia`, so the browser asks which tab or screen to share, and audio
+arrives only when "Share tab audio" (or "Share system audio") is ticked. Firefox and Safari return no
+audio from it.
## How it works
diff --git a/apps/standalone/transcription-poc/index.html b/apps/standalone/transcription-poc/index.html
index 2efd497fce..39aecf7daa 100644
--- a/apps/standalone/transcription-poc/index.html
+++ b/apps/standalone/transcription-poc/index.html
@@ -27,7 +27,8 @@
<legend>Transcribe</legend>
<select id="clip"></select>
<button id="transcribe">Transcribe clip</button>
- <button id="listen">Listen to the microphone</button>
+ <button id="listen-microphone"></button>
+ <button id="listen-tab"></button>
</fieldset>
<pre id="log"></pre>
diff --git a/apps/standalone/transcription-poc/main.js b/apps/standalone/transcription-poc/main.js
index f2a6001d33..354ab08fda 100644
--- a/apps/standalone/transcription-poc/main.js
+++ b/apps/standalone/transcription-poc/main.js
@@ -69,9 +69,32 @@ const CLIPS = [
[ "German, 4 s", `${HF}/csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3/resolve/1a468a35cbba69418f126de829e75261dea4a4e4/test_wavs/de.wav` ]
];
+/** The live sources. Each runs its own VAD in the worker and is labeled in the log. */
+const SOURCES = {
+ microphone: {
+ button: "listen-microphone",
+ label: "Listen to the microphone",
+ // Echo cancellation keeps the tab's audio, played through the speakers, out of the microphone.
+ open: () => navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true } })
+ },
+ tab: {
+ button: "listen-tab",
+ label: "Listen to a tab",
+ // Chrome requires video to share audio; the video track is never read.
+ open: () => navigator.mediaDevices.getDisplayMedia({
+ video: true,
+ audio: { echoCancellation: false, noiseSuppression: false, autoGainControl: false },
+ systemAudio: "include",
+ selfBrowserSurface: "exclude",
+ preferCurrentTab: false
+ })
+ }
+};
+
const $ = (id) => document.getElementById(id);
const worker = new Worker("worker.js");
-let microphone = null;
+/** The running captures, by source. */
+const captures = new Map();
for (const model of MODELS) {
$("model").append(new Option(model.label, model.id));
@@ -87,9 +110,10 @@ worker.onmessage = ({ data }) => {
case "downloaded": return void reportPersistence();
case "loaded": return log(`Loaded ${data.model}: copy into MEMFS ${ms(data.timings.copyToMemfsMs)}, create recognizer ${ms(data.timings.createRecognizerMs)}, WebAssembly memory ${data.heapAfterLoad} MiB.`);
case "result": return log(`${data.label}: ${ms(data.ms)} for ${ms(data.audioMs)} of audio (RTF ${data.rtf.toFixed(2)}), memory ${data.heap} MiB\n → ${data.text}`);
- case "utterance": return log(`[${ms(data.offsetMs)}] ${data.text}\n (${ms(data.ms)} for ${ms(data.audioMs)}, RTF ${data.rtf.toFixed(2)}, lag ${ms(data.lagMs)})`);
- case "live-started": return log("Listening.");
- case "live-stopped": return log("Stopped listening.");
+ case "utterance": return log(`${data.source} [${ms(data.offsetMs)}] ${data.text}\n (${ms(data.ms)} for ${ms(data.audioMs)}, RTF ${data.rtf.toFixed(2)}, lag ${ms(data.lagMs)})`);
+ case "peak": return log(`${data.source}: loudest sample ${data.peak.toFixed(4)} in the last 5 s`);
+ case "live-started": return log(`${data.source}: listening.`);
+ case "live-stopped": return log(`${data.source}: stopped listening.`);
case "deleted": return log(`Deleted ${data.model} from OPFS.`);
case "log": return console.debug(data.text);
case "error": return log(`Error: ${data.message}`);
@@ -108,19 +132,39 @@ $("transcribe").onclick = async () => {
const samples = await decodeClip(url);
worker.postMessage({ type: "transcribe", label, samples }, [ samples.buffer ]);
};
-$("listen").onclick = async () => {
- if (microphone) {
- microphone.stop();
- microphone = null;
- worker.postMessage({ type: "live-stop" });
- $("listen").textContent = "Listen to the microphone";
- return;
+for (const [ source, { button, label, open } ] of Object.entries(SOURCES)) {
+ $(button).textContent = label;
+ $(button).onclick = () => (captures.has(source) ? stopSource(source) : startSource(source, open));
+}
+
+async function startSource(source, open) {
+ let media;
+ try {
+ media = await open();
+ } catch (e) {
+ return log(`${source}: ${e.message}`);
}
- worker.postMessage({ type: "live-start" });
- microphone = await captureMicrophone((samples) =>
- worker.postMessage({ type: "samples", samples }, [ samples.buffer ]));
- $("listen").textContent = "Stop listening";
-};
+ const [ track ] = media.getAudioTracks();
+ if (!track) {
+ stopTracks(media);
+ return log(`${source}: nothing to listen to. Share a tab and tick "Share tab audio" (or "Share system audio" for a whole screen).`);
+ }
+ log(`${source}: ${track.label}, ${JSON.stringify(track.getSettings())}`);
+ // Ending the share from the browser's own bar ends the track.
+ track.addEventListener("ended", () => stopSource(source));
+
+ worker.postMessage({ type: "live-start", source });
+ captures.set(source, captureAudio(media, (samples) =>
+ worker.postMessage({ type: "samples", source, samples }, [ samples.buffer ])));
+ $(SOURCES[source].button).textContent = `Stop: ${SOURCES[source].label.toLowerCase()}`;
+}
+
+function stopSource(source) {
+ captures.get(source)?.stop();
+ captures.delete(source);
+ worker.postMessage({ type: "live-stop", source });
+ $(SOURCES[source].button).textContent = SOURCES[source].label;
+}
/** Asks the browser not to evict the models under storage pressure. Only the page can ask. */
async function reportPersistence() {
@@ -149,23 +193,28 @@ async function decodeClip(url) {
}
}
-async function captureMicrophone(onSamples) {
- const media = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } });
+/** Feeds a stream's audio, downmixed and resampled to mono 16 kHz, to `onSamples`. */
+function captureAudio(media, onSamples) {
const context = new AudioContext({ sampleRate: 16000 });
const processor = context.createScriptProcessor(4096, 1, 1);
processor.onaudioprocess = (e) => onSamples(e.inputBuffer.getChannelData(0).slice());
- context.createMediaStreamSource(media).connect(processor);
+ // Only the audio tracks: a MediaStreamSource rejects a stream whose first track is video.
+ context.createMediaStreamSource(new MediaStream(media.getAudioTracks())).connect(processor);
processor.connect(context.destination);
return {
stop() {
- for (const track of media.getTracks()) {
- track.stop();
- }
+ stopTracks(media);
void context.close();
}
};
}
+function stopTracks(media) {
+ for (const track of media.getTracks()) {
+ track.stop();
+ }
+}
+
function showProgress({ file, loaded, total, cached, ms: took }) {
const percent = Math.round(loaded / total * 100);
$("progress").textContent = cached
diff --git a/apps/standalone/transcription-poc/worker.js b/apps/standalone/transcription-poc/worker.js
index 86854944ec..fbecf38240 100644
--- a/apps/standalone/transcription-poc/worker.js
+++ b/apps/standalone/transcription-poc/worker.js
@@ -18,9 +18,8 @@ self.Module = {
importScripts("vendor/sherpa-onnx-wasm-main-vad-asr.js", "vendor/sherpa-onnx-asr.js", "vendor/sherpa-onnx-vad.js");
let recognizer = null;
-let vad = null;
-let pending = new Float32Array(0);
-let liveSamples = 0;
+/** One VAD per live source ("microphone", "tab"), so their utterances never mix. */
+const lives = new Map();
self.onmessage = async ({ data }) => {
try {
@@ -28,9 +27,9 @@ self.onmessage = async ({ data }) => {
case "download": return await download(data.model);
case "load": return await load(data.model);
case "transcribe": return transcribeClip(data.label, data.samples);
- case "live-start": return startLive();
- case "samples": return acceptLive(data.samples);
- case "live-stop": return stopLive();
+ case "live-start": return startLive(data.source);
+ case "samples": return acceptLive(data.source, data.samples);
+ case "live-stop": return stopLive(data.source);
case "delete": return await deleteModel(data.model);
}
} catch (e) {
@@ -127,10 +126,11 @@ async function load(model) {
}
function unload() {
+ for (const source of lives.keys()) {
+ stopLive(source);
+ }
recognizer?.free();
recognizer = null;
- vad?.free();
- vad = null;
}
function transcribeClip(label, samples) {
@@ -142,10 +142,10 @@ function transcribeClip(label, samples) {
post("result", { label, text, ms, audioMs, rtf: ms / audioMs, heap: heapMiB() });
}
-function startLive() {
+function startLive(source) {
requireRecognizer();
- vad?.free();
- vad = createVad(Module, {
+ lives.get(source)?.vad.free();
+ const vad = createVad(Module, {
sileroVad: {
model: "/models/silero_vad.onnx",
threshold: 0.5,
@@ -161,49 +161,63 @@ function startLive() {
debug: 0,
bufferSizeInSeconds: 60
});
- pending = new Float32Array(0);
- liveSamples = 0;
- post("live-started", {});
+ lives.set(source, { vad, pending: new Float32Array(0), samples: 0, peak: 0, peakSince: performance.now() });
+ post("live-started", { source });
}
-function acceptLive(samples) {
- if (!vad) {
+function acceptLive(source, samples) {
+ const live = lives.get(source);
+ if (!live) {
return;
}
- liveSamples += samples.length;
- const all = new Float32Array(pending.length + samples.length);
- all.set(pending);
- all.set(samples, pending.length);
+ live.samples += samples.length;
+ reportPeak(source, live, samples);
+ const all = new Float32Array(live.pending.length + samples.length);
+ all.set(live.pending);
+ all.set(samples, live.pending.length);
let offset = 0;
for (; offset + VAD_WINDOW <= all.length; offset += VAD_WINDOW) {
- vad.acceptWaveform(all.subarray(offset, offset + VAD_WINDOW));
+ live.vad.acceptWaveform(all.subarray(offset, offset + VAD_WINDOW));
}
- pending = all.slice(offset);
- drainVad();
+ live.pending = all.slice(offset);
+ drainVad(source, live);
}
-function stopLive() {
- if (!vad) {
+function stopLive(source) {
+ const live = lives.get(source);
+ if (!live) {
return;
}
- vad.flush();
- drainVad();
- vad.free();
- vad = null;
- post("live-stopped", {});
+ live.vad.flush();
+ drainVad(source, live);
+ live.vad.free();
+ lives.delete(source);
+ post("live-stopped", { source });
+}
+
+/** Logs the loudest sample every 5 s, to tell a silent capture from one without speech. */
+function reportPeak(source, live, samples) {
+ for (const sample of samples) {
+ live.peak = Math.max(live.peak, Math.abs(sample));
+ }
+ if (performance.now() - live.peakSince >= 5000) {
+ post("peak", { source, peak: live.peak });
+ live.peak = 0;
+ live.peakSince = performance.now();
+ }
}
-function drainVad() {
- while (!vad.isEmpty()) {
- const segment = vad.front();
- vad.pop();
+function drainVad(source, live) {
+ while (!live.vad.isEmpty()) {
+ const segment = live.vad.front();
+ live.vad.pop();
const started = performance.now();
const text = decode(segment.samples);
const ms = performance.now() - started;
const audioMs = segment.samples.length / SAMPLE_RATE * 1000;
// How far the transcript trails the speaker once this utterance is out.
- const lagMs = (liveSamples - segment.start) / SAMPLE_RATE * 1000 - audioMs + ms;
- post("utterance", { text, ms, audioMs, rtf: ms / audioMs, lagMs, offsetMs: segment.start / SAMPLE_RATE * 1000 });
+ const lagMs = (live.samples - segment.start) / SAMPLE_RATE * 1000 - audioMs + ms;
+ post("utterance", { source, text, ms, audioMs, rtf: ms / audioMs, lagMs, offsetMs: segment.start / SAMPLE_RATE * 1000 });
}
}