Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(desktop): transcribe system audio as a second source (PoC)
Elian Dorane0170c281ca8
diff --git a/apps/desktop/src/services/transcription.ts b/apps/desktop/src/services/transcription.tsindex bee7983ae2..d1e38626ca 100644--- a/apps/desktop/src/services/transcription.ts+++ b/apps/desktop/src/services/transcription.ts@@ -1,8 +1,9 @@import { getLog } from "@triliumnext/core";import dataDirs from "@triliumnext/server/src/services/data_dir.js";-import { app, BrowserWindow, ipcMain, session } from "electron";+import { app, BrowserWindow, desktopCapturer, ipcMain, session } from "electron";import fs from "fs";import path from "path";+import type { OfflineRecognizer, Vad } from "sherpa-onnx-node";const SAMPLE_RATE = 16000;/** Silero VAD consumes 512-sample windows at 16 kHz. */@@ -10,11 +11,19 @@ const VAD_WINDOW_SIZE = 512;const SAMPLES_CHANNEL = "transcription-poc-samples";const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");+/** The microphone carries the local speaker; the system output carries everyone else. */+const SOURCE_LABELS = {+ microphone: "Me",+ system: "Others"+} as const;++type AudioSource = keyof typeof SOURCE_LABELS;+/**- * Proof of concept: records the default microphone in a hidden window, splits it into utterances- * with Silero VAD and logs what Parakeet hears in each. Runs only when- * `<data dir>/transcription-model` holds `sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8` plus- * `silero_vad.onnx`.+ * Proof of concept: records the default microphone and the system audio output in a hidden window,+ * splits each into utterances with its own Silero VAD and logs what Parakeet hears in each,+ * labeled by source. Runs only when `<data dir>/transcription-model` holds+ * `sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8` plus `silero_vad.onnx`.*/export async function startTranscription() {const log = getLog();@@ -24,8 +33,8 @@ export async function startTranscription() {}const mod = await import("sherpa-onnx-node");- const { OfflineRecognizer, Vad } = mod.default ?? mod;- const recognizer = await OfflineRecognizer.createAsync({+ const sherpa = mod.default ?? mod;+ const recognizer = await sherpa.OfflineRecognizer.createAsync({featConfig: { sampleRate: SAMPLE_RATE, featureDim: 80 },modelConfig: {transducer: {@@ -40,7 +49,7 @@ export async function startTranscription() {},decodingMethod: "greedy_search"});- const vad = new Vad({+ const createVad = () => new sherpa.Vad({sileroVad: {model: path.join(MODEL_DIR, "silero_vad.onnx"),threshold: 0.5,@@ -54,57 +63,132 @@ export async function startTranscription() {provider: "cpu"}, 60);- // Utterances are decoded one at a time so the log keeps their order.- let decoding = Promise.resolve();- const transcribe = async (start: number, samples: Float32Array) => {- const stream = recognizer.createStream();- stream.acceptWaveform({ samples, sampleRate: SAMPLE_RATE });- const text = (await recognizer.decodeAsync(stream)).text.trim();- if (text) {- log.info(`Transcription [${formatOffset(start)}]: ${text}`);- }+ const startedAt = Date.now();+ const transcriber = new Transcriber(recognizer, startedAt);+ const pipelines = {+ microphone: new SourcePipeline("microphone", createVad(), transcriber),+ system: new SourcePipeline("system", createVad(), transcriber)};const captureWindow = await createCaptureWindow();- ipcMain.on(SAMPLES_CHANNEL, (event, samples: Float32Array) => {- if (event.sender !== captureWindow.webContents) {+ ipcMain.on(SAMPLES_CHANNEL, (event, source: AudioSource, samples: Float32Array) => {+ if (event.sender !== captureWindow.webContents || !Object.hasOwn(pipelines, source)) {return;}try {- for (let i = 0; i + VAD_WINDOW_SIZE <= samples.length; i += VAD_WINDOW_SIZE) {- vad.acceptWaveform(samples.subarray(i, i + VAD_WINDOW_SIZE));- }-- while (!vad.isEmpty()) {- const segment = vad.front(false);- vad.pop();- decoding = decoding- .then(() => transcribe(segment.start, segment.samples))- .catch((e) => log.error(`Transcription failed: ${e}`));- }+ pipelines[source].accept(samples);} catch (e) {log.error(`Transcription failed: ${e instanceof Error ? e.message : String(e)}`);}});- log.info("Transcription: listening to the microphone.");+ try {+ // getDisplayMedia() requires a user gesture, which the hidden window can only get this way.+ const tracks = await captureWindow.webContents+ .executeJavaScript("captureSystemAudio()", true);+ log.info(`Transcription: capturing system audio from ${tracks}.`);+ } catch (e) {+ const message = e instanceof Error ? e.message : String(e);+ log.error(`Transcription: system audio unavailable: ${message}`);+ }+}++/** Decodes utterances one at a time, so the log keeps their order. */+class Transcriber {+ private queue = Promise.resolve();++ constructor(+ private readonly recognizer: OfflineRecognizer,+ private readonly startedAt: number+ ) {}++ enqueue(source: AudioSource, capturedAt: number, samples: Float32Array) {+ this.queue = this.queue+ .then(() => this.transcribe(source, capturedAt, samples))+ .catch((e) => getLog().error(`Transcription failed: ${e}`));+ }++ private async transcribe(source: AudioSource, capturedAt: number, samples: Float32Array) {+ const stream = this.recognizer.createStream();+ stream.acceptWaveform({ samples, sampleRate: SAMPLE_RATE });+ const text = (await this.recognizer.decodeAsync(stream)).text.trim();+ if (text) {+ const offset = formatOffset(capturedAt - this.startedAt);+ getLog().info(`Transcription [${offset}] ${SOURCE_LABELS[source]}: ${text}`);+ }+ }+}++/**+ * Runs one source through its own VAD, so speech on one source never splits an utterance on the+ * other.+ */+class SourcePipeline {+ /** Wall-clock time of the source's first sample, which the VAD counts segment starts from. */+ private firstSampleAt: number | null = null;++ constructor(+ private readonly source: AudioSource,+ private readonly vad: Vad,+ private readonly transcriber: Transcriber+ ) {}++ private peak = 0;+ private sampleCount = 0;++ accept(samples: Float32Array) {+ this.firstSampleAt ??= Date.now() - samples.length / SAMPLE_RATE * 1000;++ for (const sample of samples) {+ this.peak = Math.max(this.peak, Math.abs(sample));+ }+ this.sampleCount += samples.length;+ if (this.sampleCount >= 5 * SAMPLE_RATE) {+ getLog().info(`Transcription: ${this.source} peak ${this.peak.toFixed(4)} over 5 s`);+ this.peak = 0;+ this.sampleCount = 0;+ }++ for (let i = 0; i + VAD_WINDOW_SIZE <= samples.length; i += VAD_WINDOW_SIZE) {+ this.vad.acceptWaveform(samples.subarray(i, i + VAD_WINDOW_SIZE));+ }++ while (!this.vad.isEmpty()) {+ const segment = this.vad.front(false);+ this.vad.pop();+ const capturedAt = this.firstSampleAt + segment.start / SAMPLE_RATE * 1000;+ this.transcriber.enqueue(this.source, capturedAt, segment.samples);+ }+ }}-function formatOffset(sampleIndex: number) {- const seconds = Math.floor(sampleIndex / SAMPLE_RATE);+function formatOffset(milliseconds: number) {+ const seconds = Math.max(0, Math.floor(milliseconds / 1000));return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;}/*** The window runs only {@link CAPTURE_PAGE}, a local file, so it gets Node integration to reach- * `ipcRenderer` without a preload. Its in-memory session grants the microphone and nothing else.+ * `ipcRenderer` without a preload. Its in-memory session grants the microphone and screen capture+ * and nothing else; every screen capture request gets the first screen plus the system audio.*/async function createCaptureWindow() {const captureSession = session.fromPartition("transcription");- captureSession.setPermissionRequestHandler((_webContents, permission, callback) => callback(permission === "media"));- captureSession.setPermissionCheckHandler((_webContents, permission) => permission === "media");+ const isAllowed = (permission: string) =>+ permission === "media" || permission === "display-capture";+ captureSession.setPermissionRequestHandler((_webContents, permission, callback) =>+ callback(isAllowed(permission)));+ captureSession.setPermissionCheckHandler((_webContents, permission) => isAllowed(permission));+ captureSession.setDisplayMediaRequestHandler((_request, callback) => {+ desktopCapturer.getSources({ types: ["screen"] }).then(([screen]) => {+ callback(screen ? { video: screen, audio: "loopback" } : {});+ }).catch((e) => {+ getLog().error(`Transcription: no screen to capture: ${e}`);+ callback({});+ });+ });const pagePath = path.join(app.getPath("temp"), "trilium-transcription.html");fs.writeFileSync(pagePath, CAPTURE_PAGE);@@ -119,7 +203,8 @@ async function createCaptureWindow() {autoplayPolicy: "no-user-gesture-required"}});- window.webContents.on("console-message", ({ message }) => getLog().info(`Transcription (capture): ${message}`));+ window.webContents.on("console-message", ({ message }) =>+ getLog().info(`Transcription (capture): ${message}`));await window.loadFile(pagePath);return window;}@@ -127,13 +212,38 @@ async function createCaptureWindow() {const CAPTURE_PAGE = `<!DOCTYPE html><script>const { ipcRenderer } = require("electron");+const context = new AudioContext({ sampleRate: ${SAMPLE_RATE} });-navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } }).then((media) => {- const context = new AudioContext({ sampleRate: ${SAMPLE_RATE} });+function pipe(source, media) {const processor = context.createScriptProcessor(4096, 1, 1);- processor.onaudioprocess = (e) => ipcRenderer.send("${SAMPLES_CHANNEL}", e.inputBuffer.getChannelData(0).slice());+ processor.onaudioprocess = (e) =>+ ipcRenderer.send("${SAMPLES_CHANNEL}", source, e.inputBuffer.getChannelData(0).slice());context.createMediaStreamSource(media).connect(processor);processor.connect(context.destination);- console.log("Capturing at " + context.sampleRate + " Hz.");+}++async function captureSystemAudio() {+ const media = await navigator.mediaDevices.getDisplayMedia({+ video: true,+ audio: { echoCancellation: false, noiseSuppression: false, autoGainControl: false }+ });+ for (const track of media.getVideoTracks()) {+ track.stop();+ }+ const audioTracks = media.getAudioTracks();+ if (!audioTracks.length) {+ throw new Error("the screen capture has no audio track");+ }+ for (const track of audioTracks) {+ console.log("System track muted=" + track.muted + " settings="+ + JSON.stringify(track.getSettings()));+ }+ pipe("system", new MediaStream(audioTracks));+ return audioTracks.map((track) => track.label).join(", ");+}++navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } }).then((media) => {+ pipe("microphone", media);+ console.log("Capturing the microphone at " + context.sampleRate + " Hz.");}).catch((e) => console.error("Microphone unavailable: " + e));</script>`;