Skip to content

Build your personal knowledge base with Trilium Notes

feat(desktop): transcribe the microphone into the log (PoC)

Elian Doran956b4c3e8a47
diff --git a/apps/desktop/package.json b/apps/desktop/package.json
index 51427b230c..abd8a9088e 100644
--- a/apps/desktop/package.json
+++ b/apps/desktop/package.json
@@ -33,7 +33,8 @@
"electron-debug": "5.0.0",
"electron-dl": "5.0.0",
"electron-squirrel-startup": "1.0.1",
- "electron-window-state": "5.0.3"
+ "electron-window-state": "5.0.3",
+ "sherpa-onnx-node": "1.13.8"
},
"devDependencies": {
"@electron-forge/cli": "8.0.1",
diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts
index a8b5d328e8..e4b73c8ad7 100644
--- a/apps/desktop/src/main.ts
+++ b/apps/desktop/src/main.ts
@@ -47,6 +47,7 @@ import { setupRestoreHandlers } from "./services/restore";
import { getSecuritySettings, registerSecurityIpcHandlers } from "./services/security_settings";
import { setupShellHandlers } from "./services/shell";
import { markStartupMetric, setupStartupMetricsIpc } from "./services/startup_metrics";
+import { startTranscription } from "./services/transcription";
import { setupSystemTray } from "./services/tray";
import windowService, { setupWindowing } from "./services/window";
@@ -366,6 +367,7 @@ async function onReady() {
// available to summon it from.
const startHidden = wasLaunchedHidden() && !options.getOptionBool("disableTray");
await windowService.createMainWindow(startHidden);
+ startTranscription().catch((e) => getLog().error(`Transcription did not start: ${e}`));
// Repair the OS autostart entry so it matches the stored option (it can
// drift if the user toggled it elsewhere). Options are loaded now that the
diff --git a/apps/desktop/src/services/transcription.ts b/apps/desktop/src/services/transcription.ts
new file mode 100644
index 0000000000..59754a45cd
--- /dev/null
+++ b/apps/desktop/src/services/transcription.ts
@@ -0,0 +1,109 @@
+import { getLog } from "@triliumnext/core";
+import dataDirs from "@triliumnext/server/src/services/data_dir.js";
+import { app, BrowserWindow, ipcMain, session } from "electron";
+import fs from "fs";
+import path from "path";
+
+const SAMPLE_RATE = 16000;
+const SAMPLES_CHANNEL = "transcription-poc-samples";
+const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");
+
+/**
+ * Proof of concept: records the default microphone in a hidden window and logs what a
+ * sherpa-onnx streaming recognizer hears. Runs only when `<data dir>/transcription-model`
+ * holds a streaming Zipformer transducer (`sherpa-onnx-streaming-zipformer-en-20M-2023-02-17`).
+ */
+export async function startTranscription() {
+ const log = getLog();
+ if (!fs.existsSync(MODEL_DIR)) {
+ log.info(`Transcription: no model in ${MODEL_DIR}, not starting.`);
+ return;
+ }
+
+ const mod = await import("sherpa-onnx-node");
+ const { OnlineRecognizer } = mod.default ?? mod;
+ const recognizer = new OnlineRecognizer({
+ featConfig: { sampleRate: SAMPLE_RATE, featureDim: 80 },
+ modelConfig: {
+ transducer: {
+ encoder: path.join(MODEL_DIR, "encoder-epoch-99-avg-1.int8.onnx"),
+ decoder: path.join(MODEL_DIR, "decoder-epoch-99-avg-1.onnx"),
+ joiner: path.join(MODEL_DIR, "joiner-epoch-99-avg-1.int8.onnx")
+ },
+ tokens: path.join(MODEL_DIR, "tokens.txt"),
+ numThreads: 2,
+ provider: "cpu"
+ },
+ decodingMethod: "greedy_search",
+ enableEndpoint: true
+ });
+ const stream = recognizer.createStream();
+
+ const captureWindow = await createCaptureWindow();
+
+ ipcMain.on(SAMPLES_CHANNEL, (event, samples: Float32Array) => {
+ if (event.sender !== captureWindow.webContents) {
+ return;
+ }
+
+ try {
+ stream.acceptWaveform({ samples, sampleRate: SAMPLE_RATE });
+ while (recognizer.isReady(stream)) {
+ recognizer.decode(stream);
+ }
+
+ if (recognizer.isEndpoint(stream)) {
+ const text = recognizer.getResult(stream).text.trim();
+ if (text) {
+ log.info(`Transcription: ${text}`);
+ }
+ recognizer.reset(stream);
+ }
+ } catch (e) {
+ log.error(`Transcription failed: ${e instanceof Error ? e.message : String(e)}`);
+ }
+ });
+
+ log.info("Transcription: listening to the microphone.");
+}
+
+/**
+ * The window runs only {@link CAPTURE_PAGE}, a local file, so it gets Node integration to reach
+ * `ipcRenderer` without a preload. Its in-memory session grants the microphone and nothing else.
+ */
+async function createCaptureWindow() {
+ const captureSession = session.fromPartition("transcription");
+ captureSession.setPermissionRequestHandler((_webContents, permission, callback) => callback(permission === "media"));
+ captureSession.setPermissionCheckHandler((_webContents, permission) => permission === "media");
+
+ const pagePath = path.join(app.getPath("temp"), "trilium-transcription.html");
+ fs.writeFileSync(pagePath, CAPTURE_PAGE);
+
+ const window = new BrowserWindow({
+ show: false,
+ webPreferences: {
+ session: captureSession,
+ nodeIntegration: true,
+ contextIsolation: false,
+ sandbox: false,
+ autoplayPolicy: "no-user-gesture-required"
+ }
+ });
+ window.webContents.on("console-message", ({ message }) => getLog().info(`Transcription (capture): ${message}`));
+ await window.loadFile(pagePath);
+ return window;
+}
+
+const CAPTURE_PAGE = `<!DOCTYPE html>
+<script>
+const { ipcRenderer } = require("electron");
+
+navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } }).then((media) => {
+ const context = new AudioContext({ sampleRate: ${SAMPLE_RATE} });
+ const processor = context.createScriptProcessor(4096, 1, 1);
+ processor.onaudioprocess = (e) => ipcRenderer.send("${SAMPLES_CHANNEL}", e.inputBuffer.getChannelData(0).slice());
+ context.createMediaStreamSource(media).connect(processor);
+ processor.connect(context.destination);
+ console.log("Capturing at " + context.sampleRate + " Hz.");
+}).catch((e) => console.error("Microphone unavailable: " + e));
+</script>`;
diff --git a/apps/desktop/src/sherpa-onnx-node.d.ts b/apps/desktop/src/sherpa-onnx-node.d.ts
new file mode 100644
index 0000000000..fac8342b9f
--- /dev/null
+++ b/apps/desktop/src/sherpa-onnx-node.d.ts
@@ -0,0 +1,15 @@
+declare module "sherpa-onnx-node" {
+ export interface OnlineStream {
+ acceptWaveform(wave: { samples: Float32Array; sampleRate: number }): void;
+ }
+
+ export class OnlineRecognizer {
+ constructor(config: object);
+ createStream(): OnlineStream;
+ isReady(stream: OnlineStream): boolean;
+ decode(stream: OnlineStream): void;
+ isEndpoint(stream: OnlineStream): boolean;
+ reset(stream: OnlineStream): void;
+ getResult(stream: OnlineStream): { text: string };
+ }
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 90048848cb..92665333f1 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -444,6 +444,9 @@ importers:
electron-window-state:
specifier: 5.0.3
version: 5.0.3
+ sherpa-onnx-node:
+ specifier: 1.13.8
+ version: 1.13.8
devDependencies:
'@electron-forge/cli':
specifier: 8.0.1
@@ -10491,6 +10494,39 @@ packages:
resolution: {integrity: sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-FPNgJMgnWVl/KhRTIhG3KL3A4Om63Rn4YKXc9/uHY7SzLcvqLJLc/h7UBWJwduXvv7K18t5NpxHR6XgXn4sjWw==}
+ cpu: [arm64]
+ os: [darwin]
+
+ resolution: {integrity: sha512-7BLRpjM6w4f9W46/nmkmq8lEKUayhebvcpslCVQ+6QN2uReYlZEMDZlSpXMjme+hUFrPfRz8P3UNq8ep/4d19g==}
+ cpu: [x64]
+ os: [darwin]
+
+ resolution: {integrity: sha512-Tlg7a70b/Wge3OF8IgTHF9jhSVCsLyKQKhwc4BsJ5A+dL/SrFtGBjzuHp4XeLhiiOT7afCxX5PdSn/D4c8Lnuw==}
+ cpu: [arm64]
+ os: [linux]
+
+ resolution: {integrity: sha512-6plnhjagsSeTntCgnlag86hWbs/uZE9Crms1LgOb68/1nKsIQjMd+WG519m+aPwT6TrsBOiEMzrx41t8sL5L5g==}
+ cpu: [x64]
+ os: [linux]
+
+ resolution: {integrity: sha512-MsDMBdhLFTZ1GwvcGSSQhnS7g/EA8OMH6IYysCVUOM7j8Icty9KRc0E6YT1A5fWBsZwRfKOeh88QC95aRvS8ag==}
+
+ resolution: {integrity: sha512-H0Ojln9hfvM+pxWqW0cnjB/XK8/JPq4/k0IWu2JpY5Z458M3zv+XC3Hu+wmot3AoH/K1Bgt23n/tTKtPV9x8qg==}
+ cpu: [ia32]
+ os: [win32]
+
+ resolution: {integrity: sha512-oZF1c9VPOKtMwn83Bboc5XSWL+76BRoyB3eUuVnCknBKxwSULZU2Foia9VHWzU+n4I12rPsP6z6H9Rp1hD9o8g==}
+ cpu: [x64]
+ os: [win32]
+
resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==}
engines: {node: '>= 0.4'}
@@ -20616,7 +20652,7 @@ snapshots:
dependencies:
- '@types/node': 24.19.1
+ '@types/node': 24.19.2
merge-stream: 2.0.0
supports-color: 8.1.1
optional: true
@@ -23071,6 +23107,33 @@ snapshots:
+ optional: true
+
+ optional: true
+
+ optional: true
+
+ optional: true
+
+ optionalDependencies:
+ sherpa-onnx-darwin-arm64: 1.13.8
+ sherpa-onnx-darwin-x64: 1.13.8
+ sherpa-onnx-linux-arm64: 1.13.8
+ sherpa-onnx-linux-x64: 1.13.8
+ sherpa-onnx-win-ia32: 1.13.8
+ sherpa-onnx-win-x64: 1.13.8
+
+ optional: true
+
+ optional: true
+
dependencies:
es-errors: 1.3.0