Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(desktop): transcribe the microphone into the log (PoC)
Elian Doran956b4c3e8a47
diff --git a/apps/desktop/package.json b/apps/desktop/package.jsonindex 51427b230c..abd8a9088e 100644--- a/apps/desktop/package.json+++ b/apps/desktop/package.json@@ -33,7 +33,8 @@"electron-debug": "5.0.0","electron-dl": "5.0.0","electron-squirrel-startup": "1.0.1",- "electron-window-state": "5.0.3"+ "electron-window-state": "5.0.3",+ "sherpa-onnx-node": "1.13.8"},"devDependencies": {"@electron-forge/cli": "8.0.1",diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.tsindex a8b5d328e8..e4b73c8ad7 100644--- a/apps/desktop/src/main.ts+++ b/apps/desktop/src/main.ts@@ -47,6 +47,7 @@ import { setupRestoreHandlers } from "./services/restore";import { getSecuritySettings, registerSecurityIpcHandlers } from "./services/security_settings";import { setupShellHandlers } from "./services/shell";import { markStartupMetric, setupStartupMetricsIpc } from "./services/startup_metrics";+import { startTranscription } from "./services/transcription";import { setupSystemTray } from "./services/tray";import windowService, { setupWindowing } from "./services/window";@@ -366,6 +367,7 @@ async function onReady() {// available to summon it from.const startHidden = wasLaunchedHidden() && !options.getOptionBool("disableTray");await windowService.createMainWindow(startHidden);+ startTranscription().catch((e) => getLog().error(`Transcription did not start: ${e}`));// Repair the OS autostart entry so it matches the stored option (it can// drift if the user toggled it elsewhere). Options are loaded now that thediff --git a/apps/desktop/src/services/transcription.ts b/apps/desktop/src/services/transcription.tsnew file mode 100644index 0000000000..59754a45cd--- /dev/null+++ b/apps/desktop/src/services/transcription.ts@@ -0,0 +1,109 @@+import { getLog } from "@triliumnext/core";+import dataDirs from "@triliumnext/server/src/services/data_dir.js";+import { app, BrowserWindow, ipcMain, session } from "electron";+import fs from "fs";+import path from "path";++const SAMPLE_RATE = 16000;+const SAMPLES_CHANNEL = "transcription-poc-samples";+const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");++/**+ * Proof of concept: records the default microphone in a hidden window and logs what a+ * sherpa-onnx streaming recognizer hears. Runs only when `<data dir>/transcription-model`+ * holds a streaming Zipformer transducer (`sherpa-onnx-streaming-zipformer-en-20M-2023-02-17`).+ */+export async function startTranscription() {+ const log = getLog();+ if (!fs.existsSync(MODEL_DIR)) {+ log.info(`Transcription: no model in ${MODEL_DIR}, not starting.`);+ return;+ }++ const mod = await import("sherpa-onnx-node");+ const { OnlineRecognizer } = mod.default ?? mod;+ const recognizer = new OnlineRecognizer({+ featConfig: { sampleRate: SAMPLE_RATE, featureDim: 80 },+ modelConfig: {+ transducer: {+ encoder: path.join(MODEL_DIR, "encoder-epoch-99-avg-1.int8.onnx"),+ decoder: path.join(MODEL_DIR, "decoder-epoch-99-avg-1.onnx"),+ joiner: path.join(MODEL_DIR, "joiner-epoch-99-avg-1.int8.onnx")+ },+ tokens: path.join(MODEL_DIR, "tokens.txt"),+ numThreads: 2,+ provider: "cpu"+ },+ decodingMethod: "greedy_search",+ enableEndpoint: true+ });+ const stream = recognizer.createStream();++ const captureWindow = await createCaptureWindow();++ ipcMain.on(SAMPLES_CHANNEL, (event, samples: Float32Array) => {+ if (event.sender !== captureWindow.webContents) {+ return;+ }++ try {+ stream.acceptWaveform({ samples, sampleRate: SAMPLE_RATE });+ while (recognizer.isReady(stream)) {+ recognizer.decode(stream);+ }++ if (recognizer.isEndpoint(stream)) {+ const text = recognizer.getResult(stream).text.trim();+ if (text) {+ log.info(`Transcription: ${text}`);+ }+ recognizer.reset(stream);+ }+ } catch (e) {+ log.error(`Transcription failed: ${e instanceof Error ? e.message : String(e)}`);+ }+ });++ log.info("Transcription: listening to the microphone.");+}++/**+ * The window runs only {@link CAPTURE_PAGE}, a local file, so it gets Node integration to reach+ * `ipcRenderer` without a preload. Its in-memory session grants the microphone and nothing else.+ */+async function createCaptureWindow() {+ const captureSession = session.fromPartition("transcription");+ captureSession.setPermissionRequestHandler((_webContents, permission, callback) => callback(permission === "media"));+ captureSession.setPermissionCheckHandler((_webContents, permission) => permission === "media");++ const pagePath = path.join(app.getPath("temp"), "trilium-transcription.html");+ fs.writeFileSync(pagePath, CAPTURE_PAGE);++ const window = new BrowserWindow({+ show: false,+ webPreferences: {+ session: captureSession,+ nodeIntegration: true,+ contextIsolation: false,+ sandbox: false,+ autoplayPolicy: "no-user-gesture-required"+ }+ });+ window.webContents.on("console-message", ({ message }) => getLog().info(`Transcription (capture): ${message}`));+ await window.loadFile(pagePath);+ return window;+}++const CAPTURE_PAGE = `<!DOCTYPE html>+<script>+const { ipcRenderer } = require("electron");++navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } }).then((media) => {+ const context = new AudioContext({ sampleRate: ${SAMPLE_RATE} });+ const processor = context.createScriptProcessor(4096, 1, 1);+ processor.onaudioprocess = (e) => ipcRenderer.send("${SAMPLES_CHANNEL}", e.inputBuffer.getChannelData(0).slice());+ context.createMediaStreamSource(media).connect(processor);+ processor.connect(context.destination);+ console.log("Capturing at " + context.sampleRate + " Hz.");+}).catch((e) => console.error("Microphone unavailable: " + e));+</script>`;diff --git a/apps/desktop/src/sherpa-onnx-node.d.ts b/apps/desktop/src/sherpa-onnx-node.d.tsnew file mode 100644index 0000000000..fac8342b9f--- /dev/null+++ b/apps/desktop/src/sherpa-onnx-node.d.ts@@ -0,0 +1,15 @@+declare module "sherpa-onnx-node" {+ export interface OnlineStream {+ acceptWaveform(wave: { samples: Float32Array; sampleRate: number }): void;+ }++ export class OnlineRecognizer {+ constructor(config: object);+ createStream(): OnlineStream;+ isReady(stream: OnlineStream): boolean;+ decode(stream: OnlineStream): void;+ isEndpoint(stream: OnlineStream): boolean;+ reset(stream: OnlineStream): void;+ getResult(stream: OnlineStream): { text: string };+ }+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 90048848cb..92665333f1 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -444,6 +444,9 @@ importers:electron-window-state:specifier: 5.0.3version: 5.0.3+ sherpa-onnx-node:+ specifier: 1.13.8+ version: 1.13.8devDependencies:'@electron-forge/cli':specifier: 8.0.1@@ -10491,6 +10494,39 @@ packages:resolution: {integrity: sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-FPNgJMgnWVl/KhRTIhG3KL3A4Om63Rn4YKXc9/uHY7SzLcvqLJLc/h7UBWJwduXvv7K18t5NpxHR6XgXn4sjWw==}+ cpu: [arm64]+ os: [darwin]++ resolution: {integrity: sha512-7BLRpjM6w4f9W46/nmkmq8lEKUayhebvcpslCVQ+6QN2uReYlZEMDZlSpXMjme+hUFrPfRz8P3UNq8ep/4d19g==}+ cpu: [x64]+ os: [darwin]++ resolution: {integrity: sha512-Tlg7a70b/Wge3OF8IgTHF9jhSVCsLyKQKhwc4BsJ5A+dL/SrFtGBjzuHp4XeLhiiOT7afCxX5PdSn/D4c8Lnuw==}+ cpu: [arm64]+ os: [linux]++ resolution: {integrity: sha512-6plnhjagsSeTntCgnlag86hWbs/uZE9Crms1LgOb68/1nKsIQjMd+WG519m+aPwT6TrsBOiEMzrx41t8sL5L5g==}+ cpu: [x64]+ os: [linux]++ resolution: {integrity: sha512-MsDMBdhLFTZ1GwvcGSSQhnS7g/EA8OMH6IYysCVUOM7j8Icty9KRc0E6YT1A5fWBsZwRfKOeh88QC95aRvS8ag==}++ resolution: {integrity: sha512-H0Ojln9hfvM+pxWqW0cnjB/XK8/JPq4/k0IWu2JpY5Z458M3zv+XC3Hu+wmot3AoH/K1Bgt23n/tTKtPV9x8qg==}+ cpu: [ia32]+ os: [win32]++ resolution: {integrity: sha512-oZF1c9VPOKtMwn83Bboc5XSWL+76BRoyB3eUuVnCknBKxwSULZU2Foia9VHWzU+n4I12rPsP6z6H9Rp1hD9o8g==}+ cpu: [x64]+ os: [win32]+resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==}engines: {node: '>= 0.4'}@@ -20616,7 +20652,7 @@ snapshots:dependencies:- '@types/node': 24.19.1+ '@types/node': 24.19.2merge-stream: 2.0.0supports-color: 8.1.1optional: true@@ -23071,6 +23107,33 @@ snapshots:+ optional: true++ optional: true++ optional: true++ optional: true++ optionalDependencies:+ sherpa-onnx-darwin-arm64: 1.13.8+ sherpa-onnx-darwin-x64: 1.13.8+ sherpa-onnx-linux-arm64: 1.13.8+ sherpa-onnx-linux-x64: 1.13.8+ sherpa-onnx-win-ia32: 1.13.8+ sherpa-onnx-win-x64: 1.13.8++ optional: true++ optional: true+dependencies:es-errors: 1.3.0