Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(desktop): transcribe meetings in a utility process
Elian Doran8e9ef5d170be
diff --git a/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsx b/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsxindex fb334d388b..95f6aa8a95 100644--- a/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsx+++ b/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsx@@ -1,5 +1,10 @@+import type { TranscriptionState } from "@triliumnext/commons";+import { act } from "preact/test-utils";import { beforeEach, describe, expect, it, vi } from "vitest";+import type FNote from "../../entities/fnote";+import { renderInto } from "../../test/render";+const mocks = vi.hoisted(() => ({getInboxNote: vi.fn(),createNote: vi.fn(),@@ -18,8 +23,12 @@ vi.mock("../../services/toast", () => ({vi.mock("../../services/i18n", () => ({t: (key: string, options?: { date?: string }) => `${key} ${options?.date}`}));+vi.mock("./launch_bar_widgets", () => ({+ LaunchBarActionButton: () => null,+ useLauncherIconAndTitle: () => ({ icon: "bx bx-microphone", title: "Record meeting" })+}));-import { createMeetingNote, toggleRecording } from "./RecordMeetingButton";+import RecordMeetingButton, { createMeetingNote, toggleRecording } from "./RecordMeetingButton";beforeEach(() => {vi.clearAllMocks();@@ -88,3 +97,34 @@ describe("toggleRecording", () => {expect(mocks.createNote).not.toHaveBeenCalled();});});++describe("RecordMeetingButton", () => {+ it("says once why a recording stopped by itself", async () => {+ let onStateChanged: (state: TranscriptionState) => void = () => {};+ const report = (state: Partial<TranscriptionState>) => act(() => onStateChanged({+ status: "idle", noteId: null, pendingSources: [], ...state+ }));+ window.electronApi = {+ transcription: {+ getState: async () => ({ status: "idle", noteId: null, pendingSources: [] }),+ onStateChanged: (callback: typeof onStateChanged) => {+ onStateChanged = callback;+ return () => {};+ }+ }+ } as unknown as typeof window.electronApi;+ act(() => {+ renderInto(<RecordMeetingButton launcherNote={{} as FNote} />);+ });++ await report({ status: "recording", noteId: "meeting" });+ await report({ error: "Stopped unexpectedly" });+ await report({ error: "Stopped unexpectedly", pendingSources: [] });+ expect(mocks.showError).toHaveBeenCalledExactlyOnceWith("Stopped unexpectedly");++ await report({ status: "recording", noteId: "meeting" });+ await report({ error: "Stopped unexpectedly" });+ expect(mocks.showError).toHaveBeenCalledTimes(2);+ window.electronApi = undefined;+ });+});diff --git a/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx b/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsxindex 90dfa39808..34b18e7fd7 100644--- a/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx+++ b/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx@@ -21,11 +21,17 @@ import {*/export default function RecordMeetingButton({ launcherNote }: LauncherNoteProps) {const { icon, title } = useLauncherIconAndTitle(launcherNote);- const { status } = useTranscriptionState();+ const { status, error } = useTranscriptionState();const handleClick = useCallback(() => {void toggleRecording(status);}, [ status ]);+ useEffect(() => {+ if (error) {+ toast.showError(error);+ }+ }, [ error ]);+return (<LaunchBarActionButtonlauncherNote={launcherNote}diff --git a/apps/desktop/scripts/build.ts b/apps/desktop/scripts/build.tsindex ffc2202a7a..a03ed0dfdb 100644--- a/apps/desktop/scripts/build.ts+++ b/apps/desktop/scripts/build.ts@@ -20,6 +20,9 @@ async function main() {// The image compression worker, which lives in the server it embeds. Built here too so the// desktop app can compress off-thread rather than falling back to doing it in the main process.await build.buildBackend([ "../server/src/services/image_worker.ts" ]);+ // The meeting transcription process, which transcription/worker_client.ts starts by its .cjs+ // path as an Electron utility process.+ await build.buildBackend([ "src/services/transcription/transcription_worker.ts" ]);await build.buildTesseractWorker("../server/src/services/ocr/tesseract_worker.ts");// Copy assets.diff --git a/apps/desktop/src/services/transcription/audio_recorder.spec.ts b/apps/desktop/src/services/transcription/audio_recorder.spec.tsindex 907d094027..5b2a401c7a 100644--- a/apps/desktop/src/services/transcription/audio_recorder.spec.ts+++ b/apps/desktop/src/services/transcription/audio_recorder.spec.ts@@ -4,15 +4,10 @@ import path from "path";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import {- encodeWavHeader, getRecordingDirectory, type RecordingManifest, toPcm16, TranscriptFileSink,- WavRecorder+ encodeWavHeader, type RecordingManifest, toPcm16, TranscriptFileSink, WavRecorder} from "./audio_recorder";-const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));-vi.mock("@triliumnext/core", () => ({ getLog: () => log }));-vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({- default: { TRILIUM_DATA_DIR: "/data" }-}));+const log = { info: vi.fn(), error: vi.fn() };const STARTED_AT = Date.parse("2026-10-14T09:30:00.000Z");const SETTINGS = { recognizer: { decodingMethod: "greedy_search" } };@@ -35,16 +30,9 @@ function readJson<T>(fileName: string) {return JSON.parse(fs.readFileSync(path.join(directory, fileName), "utf8")) as T;}-it("names a recording's directory after when it was created and its note", () => {- expect(path.basename(getRecordingDirectory(STARTED_AT, "meeting")))- .toBe("2026-10-14T09-30-00-000Z_meeting");- expect(path.dirname(getRecordingDirectory(STARTED_AT, "meeting")))- .toBe(path.join("/data", "transcription-recordings"));-});-describe("WavRecorder", () => {it("keeps each source in a WAV file, lined up by the manifest", () => {- const recorder = new WavRecorder(directory, "meeting", SETTINGS);+ const recorder = new WavRecorder(directory, "meeting", SETTINGS, log);recorder.open(STARTED_AT);recorder.write("microphone", new Float32Array([ 0.5, -0.5 ]), 1500);recorder.write("system", new Float32Array([ 1 ]), 3000);@@ -70,7 +58,7 @@ describe("WavRecorder", () => {});it("stops keeping the audio after a failed write, logging it once", () => {- const recorder = new WavRecorder(directory, "meeting", SETTINGS);+ const recorder = new WavRecorder(directory, "meeting", SETTINGS, log);recorder.open(STARTED_AT);fs.rmSync(directory, { recursive: true });diff --git a/apps/desktop/src/services/transcription/audio_recorder.ts b/apps/desktop/src/services/transcription/audio_recorder.tsindex ea4de9b2bd..992c8304cd 100644--- a/apps/desktop/src/services/transcription/audio_recorder.ts+++ b/apps/desktop/src/services/transcription/audio_recorder.ts@@ -1,24 +1,13 @@import type { Transcript, TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";-import { getLog } from "@triliumnext/core";-import dataDirs from "@triliumnext/server/src/services/data_dir.js";import fs from "fs";import path from "path";-import { SAMPLE_RATE } from "./engine";+import { type Logger, SAMPLE_RATE } from "./protocol";import type { AudioTap, TranscriptSink } from "./session";-/** Where development builds keep the audio of each recording, one directory per recording. */-export const RECORDINGS_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-recordings");-/** The transcript the recording produced live, beside the audio it came from. */export const TRANSCRIPT_FILE = "transcript.json";-/** The directory under {@link RECORDINGS_DIR} for a recording of `noteId` created at `createdAt`. */-export function getRecordingDirectory(createdAt: number, noteId: string) {- const timestamp = new Date(createdAt).toISOString().replace(/[:.]/g, "-");- return path.join(RECORDINGS_DIR, `${timestamp}_${noteId}`);-}-/** The `recording.json` next to the audio files, which lines the sources up with the transcript. */export interface RecordingManifest {noteId: string;@@ -47,7 +36,8 @@ export class WavRecorder implements AudioTap {constructor(private readonly directory: string,private readonly noteId: string,- private readonly engineSettings: Record<string, unknown>+ private readonly engineSettings: Record<string, unknown>,+ private readonly log: Logger) {}open(startedAt: number) {@@ -60,7 +50,7 @@ export class WavRecorder implements AudioTap {sources: {}};this.writeManifest();- getLog().info(`Transcription: keeping the audio in ${this.directory}`);+ this.log.info(`Transcription: keeping the audio in ${this.directory}`);}write(source: TranscriptSource, samples: Float32Array, firstSampleOffsetMs: number) {@@ -82,7 +72,7 @@ export class WavRecorder implements AudioTap {} catch (e) {// Stops for good, so a full disk logs once rather than on every chunk.this.failed = true;- getLog().error(`Transcription: stopped keeping the audio: ${e}`);+ this.log.error(`Transcription: stopped keeping the audio: ${e}`);}}diff --git a/apps/desktop/src/services/transcription/capture.ts b/apps/desktop/src/services/transcription/capture.tsindex 50a4100d5e..b7442c2c34 100644--- a/apps/desktop/src/services/transcription/capture.ts+++ b/apps/desktop/src/services/transcription/capture.ts@@ -1,27 +1,14 @@import type { TranscriptSource } from "@triliumnext/commons";import { getLog } from "@triliumnext/core";-import { app, BrowserWindow, ipcMain, type IpcMainEvent, session } from "electron";+import { app, BrowserWindow, type MessagePortMain, session } from "electron";import fs from "fs";import path from "path";-import { SAMPLE_RATE } from "./engine";+import { SAMPLE_RATE } from "./protocol";-export type SamplesListener = (source: TranscriptSource, samples: Float32Array) => void;+const PORT_CHANNEL = "transcription-capture-port";-/** Produces mono {@link SAMPLE_RATE} samples per source until stopped. */-export interface AudioCapture {- /**- * Starts every source it can and returns the ones that run.- *- * @throws when no source could start.- */- start(onSamples: SamplesListener): Promise<TranscriptSource[]>;- stop(): void;-}--const SAMPLES_CHANNEL = "transcription-samples";--/** Samples per IPC message. `VadSegmenter` drops what does not fill a 512-sample window. */+/** Samples per message. `VadSegmenter` drops what does not fill a 512-sample window. */const CHUNK_SAMPLES = 4096;/** The page function that starts each source. */@@ -30,27 +17,23 @@ const SOURCE_STARTERS = [[ "system", "captureSystemAudio()" ]] as const;-type SamplesIpcListener =- (event: IpcMainEvent, source: TranscriptSource, samples: Float32Array) => void;-/**- * Captures the default microphone and the system audio output in a hidden window, which streams- * the samples to the main process over IPC.+ * Captures the default microphone and the system audio output in a hidden window, which posts+ * them to a port as `CaptureChunk`s of {@link CHUNK_SAMPLES} mono samples. The samples never+ * pass through the main process.*/-export class ElectronAudioCapture implements AudioCapture {+export class CaptureWindow {private window: BrowserWindow | null = null;- private listener: SamplesIpcListener | null = null;- async start(onSamples: SamplesListener) {+ /**+ * Starts every source it can, posting to `port`, and returns the ones that run.+ *+ * @throws when no source could start.+ */+ async start(port: MessagePortMain) {const window = await createCaptureWindow();this.window = window;- this.listener = (event, source, samples) => {- const isSource = source === "microphone" || source === "system";- if (event.sender === window.webContents && isSource) {- onSamples(source, samples);- }- };- ipcMain.on(SAMPLES_CHANNEL, this.listener);+ window.webContents.postMessage(PORT_CHANNEL, null, [ port ]);const sources: TranscriptSource[] = [];for (const [ source, call ] of SOURCE_STARTERS) {@@ -74,10 +57,6 @@ export class ElectronAudioCapture implements AudioCapture {}stop() {- if (this.listener) {- ipcMain.removeListener(SAMPLES_CHANNEL, this.listener);- this.listener = null;- }if (this.window && !this.window.isDestroyed()) {this.window.destroy();}@@ -87,8 +66,9 @@ export class ElectronAudioCapture implements AudioCapture {/*** The window runs only {@link CAPTURE_PAGE}, a local file, so it gets Node integration to reach- * `ipcRenderer` without a preload. Its in-memory session grants the microphone and screen capture- * and nothing else; every screen capture request gets the window itself plus the system audio.+ * `ipcRenderer`, which delivers the port, without a preload. Its in-memory session grants the+ * microphone and screen capture and nothing else; every screen capture request gets the window+ * itself plus the system audio.*/async function createCaptureWindow() {const captureSession = session.fromPartition("transcription");@@ -163,12 +143,14 @@ registerProcessor("chunk-collector", ChunkCollector);const CAPTURE_PAGE = `<!DOCTYPE html><script>const { ipcRenderer } = require("electron");+const portReceived = new Promise((resolve) =>+ ipcRenderer.once("${PORT_CHANNEL}", (e) => resolve(e.ports[0])));const context = new AudioContext({ sampleRate: ${SAMPLE_RATE} });const processorLoaded = context.audioWorklet.addModule(URL.createObjectURL(new Blob([ ${JSON.stringify(CAPTURE_PROCESSOR)} ], { type: "text/javascript" })));async function pipe(source, media) {- await processorLoaded;+ const [ port ] = await Promise.all([ portReceived, processorLoaded ]);const collector = new AudioWorkletNode(context, "chunk-collector", {numberOfInputs: 1,numberOfOutputs: 0,@@ -176,7 +158,9 @@ async function pipe(source, media) {channelCountMode: "explicit",processorOptions: { chunkSamples: ${CHUNK_SAMPLES} }});- collector.port.onmessage = (e) => ipcRenderer.send("${SAMPLES_CHANNEL}", source, e.data);+ // Copied, not transferred: a MessagePortMain receives a message with a transferred buffer+ // as null.+ collector.port.onmessage = (e) => port.postMessage({ source, samples: e.data });context.createMediaStreamSource(media).connect(collector);}diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.tsindex 3112a9d6b0..fe8a5fc6ef 100644--- a/apps/desktop/src/services/transcription/engine.ts+++ b/apps/desktop/src/services/transcription/engine.ts@@ -1,13 +1,10 @@-import dataDirs from "@triliumnext/server/src/services/data_dir.js";import fs from "fs";import path from "path";import type { SpeakerEmbeddingExtractor, Vad } from "sherpa-onnx-node";+import { SAMPLE_RATE } from "./protocol";import { SpeakerClusterer, type SpeakerClustererOptions } from "./speakers";-/** Every source and engine works on mono 16 kHz samples. */-export const SAMPLE_RATE = 16000;-/** Turns one utterance into text. */export interface SpeechRecognizer {transcribe(samples: Float32Array): Promise<string>;@@ -47,11 +44,8 @@ export interface SpeechEngine {settings: Record<string, unknown>;}-/** Where the models live: Parakeet TDT 0.6B v3 (int8) plus `silero_vad.onnx`. */-export const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");-/** The speaker embedding model, optional: without it, utterances carry no speaker. */-const SPEAKER_MODEL = path.join(MODEL_DIR, "speaker", "nemo_en_titanet_small.onnx");+const SPEAKER_MODEL = path.join("speaker", "nemo_en_titanet_small.onnx");/** Silero VAD consumes 512-sample windows at 16 kHz. */const VAD_WINDOW_SIZE = 512;@@ -85,33 +79,31 @@ const VAD_SETTINGS = {};/**- * Loads the local engine: Silero VAD for segmentation and Parakeet for recognition, both through- * sherpa-onnx. Recognition runs on the libuv thread pool, so it never blocks the main process.+ * Loads the local engine from `modelDir`: Silero VAD for segmentation (`silero_vad.onnx`) and+ * Parakeet TDT 0.6B v3 int8 for recognition, both through sherpa-onnx. Recognition runs on the+ * libuv thread pool; the VAD and the speaker embeddings run on the calling thread.*/-export async function loadSherpaEngine(): Promise<SpeechEngine> {- if (!fs.existsSync(MODEL_DIR)) {- throw new ModelMissingError(MODEL_DIR);- }-+export async function loadSherpaEngine(modelDir: string): Promise<SpeechEngine> {+ const speakerModel = path.join(modelDir, SPEAKER_MODEL);const mod = await import("sherpa-onnx-node");const sherpa = mod.default ?? mod;const recognizer = await sherpa.OfflineRecognizer.createAsync({featConfig: { sampleRate: SAMPLE_RATE, featureDim: 80 },modelConfig: {transducer: {- encoder: path.join(MODEL_DIR, "encoder.int8.onnx"),- decoder: path.join(MODEL_DIR, "decoder.int8.onnx"),- joiner: path.join(MODEL_DIR, "joiner.int8.onnx")+ encoder: path.join(modelDir, "encoder.int8.onnx"),+ decoder: path.join(modelDir, "decoder.int8.onnx"),+ joiner: path.join(modelDir, "joiner.int8.onnx")},- tokens: path.join(MODEL_DIR, "tokens.txt"),+ tokens: path.join(modelDir, "tokens.txt"),...RECOGNIZER_SETTINGS},...DECODING_SETTINGS});- const speakerExtractor = fs.existsSync(SPEAKER_MODEL)+ const speakerExtractor = fs.existsSync(speakerModel)? new sherpa.SpeakerEmbeddingExtractor({- model: SPEAKER_MODEL, numThreads: 1, provider: "cpu"+ model: speakerModel, numThreads: 1, provider: "cpu"}): null;@@ -125,7 +117,7 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {},createSegmenter: () => new VadSegmenter(new sherpa.Vad({sileroVad: {- model: path.join(MODEL_DIR, "silero_vad.onnx"),+ model: path.join(modelDir, "silero_vad.onnx"),...VAD_SETTINGS},sampleRate: SAMPLE_RATE,@@ -136,22 +128,16 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {? () => new EmbeddingSpeakerTracker(speakerExtractor, SPEAKER_SETTINGS): undefined,settings: {- modelDirectory: path.resolve(MODEL_DIR),+ modelDirectory: path.resolve(modelDir),recognizer: { ...RECOGNIZER_SETTINGS, ...DECODING_SETTINGS },vad: VAD_SETTINGS,speakers: speakerExtractor- ? { model: path.basename(SPEAKER_MODEL), ...SPEAKER_SETTINGS }+ ? { model: path.basename(speakerModel), ...SPEAKER_SETTINGS }: null}};}-export class ModelMissingError extends Error {- constructor(readonly directory: string) {- super(`No speech model in ${directory}`);- }-}-class VadSegmenter implements SpeechSegmenter {constructor(private readonly vad: Vad) {}diff --git a/apps/desktop/src/services/transcription/index.spec.ts b/apps/desktop/src/services/transcription/index.spec.tsindex 9cd5bfdc2a..adb5e5ed3f 100644--- a/apps/desktop/src/services/transcription/index.spec.ts+++ b/apps/desktop/src/services/transcription/index.spec.ts@@ -1,17 +1,19 @@import type { TranscriptionState } from "@triliumnext/commons";import { beforeEach, describe, expect, it, vi } from "vitest";-import type { RecordingSessionOptions } from "./session";+import type { WorkerListeners } from "./worker_client";const h = vi.hoisted(() => ({handlers: new Map<string, (...args: unknown[]) => unknown>(),broadcast: [] as TranscriptionState[],notes: new Set<string>([ "meeting" ]),isDev: false,- loadEngine: vi.fn(),- sessionOptions: null as RecordingSessionOptions | null,- sessionStart: vi.fn(),- sessionStop: vi.fn()+ isModelInstalled: true,+ log: { info: vi.fn(), error: vi.fn() },+ listeners: null as WorkerListeners | null,+ worker: { start: vi.fn(), connectCapture: vi.fn(), stop: vi.fn(), kill: vi.fn() },+ capture: { start: vi.fn(), stop: vi.fn() },+ sink: { open: vi.fn(), write: vi.fn() }}));vi.mock("electron", () => ({@@ -29,11 +31,16 @@ vi.mock("electron", () => ({}}}))+ },+ MessageChannelMain: class {+ port1 = "capture end";+ port2 = "worker end";}}));+vi.mock("fs", () => ({ default: { existsSync: () => h.isModelInstalled } }));vi.mock("@triliumnext/core", () => ({becca: { getNote: (noteId: string) => h.notes.has(noteId) ? { noteId } : null },- getLog: () => ({ info: vi.fn(), error: vi.fn() }),+ getLog: () => h.log,utils: { isDev: () => h.isDev }}));vi.mock("i18next", () => ({@@ -42,37 +49,35 @@ vi.mock("i18next", () => ({vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({default: { TRILIUM_DATA_DIR: "/data" }}));-vi.mock("./capture", () => ({ ElectronAudioCapture: class {} }));-vi.mock("./engine", async (importOriginal) => ({- ...(await importOriginal<typeof import("./engine")>()),- loadSherpaEngine: h.loadEngine-}));-vi.mock("./attachment_sink", () => ({ AttachmentSink: class {} }));-vi.mock("./audio_recorder", () => ({- getRecordingDirectory: (createdAt: number, noteId: string) => `/recordings/${noteId}`,- TranscriptFileSink: class {- constructor(readonly directory: string) {}- },- WavRecorder: class {- constructor(- readonly directory: string, readonly noteId: string, readonly settings: object- ) {}+vi.mock("./capture", () => ({+ CaptureWindow: class {+ start = h.capture.start;+ stop = h.capture.stop;}}));-vi.mock("./session", () => ({- RecordingSession: class {- start = h.sessionStart;- stop = h.sessionStop;+vi.mock("./attachment_sink", () => ({+ AttachmentSink: class {+ open = h.sink.open;+ write = h.sink.write;- constructor(options: RecordingSessionOptions) {- h.sessionOptions = options;+ constructor(readonly noteId: string) {}+ }+}));+vi.mock("./worker_client", () => ({+ TranscriptionWorker: class {+ start = h.worker.start;+ connectCapture = h.worker.connectCapture;+ stop = h.worker.stop;+ kill = h.worker.kill;++ constructor(listeners: WorkerListeners) {+ h.listeners = listeners;}}}));-const { ModelMissingError } = await import("./engine");-const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };+const STARTED_AT = Date.parse("2026-10-14T09:30:00.000Z");async function loadService() {vi.resetModules();@@ -86,8 +91,10 @@ beforeEach(() => {h.handlers.clear();h.broadcast.length = 0;h.isDev = false;- h.loadEngine.mockResolvedValue({});- h.sessionStart.mockResolvedValue([ "microphone", "system" ]);+ h.isModelInstalled = true;+ h.listeners = null;+ h.worker.start.mockResolvedValue({ type: "ready", startedAt: STARTED_AT, settings: {} });+ h.capture.start.mockResolvedValue([ "microphone", "system" ]);});describe("transcription IPC", () => {@@ -101,7 +108,8 @@ describe("transcription IPC", () => {.toEqual({ status: "recording", noteId: "meeting", pendingSources: [] });await invoke("transcription-stop");- expect(h.sessionStop).toHaveBeenCalledOnce();+ expect(h.capture.stop.mock.invocationCallOrder[0])+ .toBeLessThan(h.worker.stop.mock.invocationCallOrder[0]);expect(h.broadcast.map((state) => state.status)).toEqual(["starting", "starting", "recording", "recording","stopping", "stopping", "idle", "idle"@@ -109,11 +117,37 @@ describe("transcription IPC", () => {expect(await invoke("transcription-get-state")).toEqual(IDLE);});- it("tells every window which sources are pending while it records", async () => {+ it("connects the capture window to the transcription process once that is ready", async () => {+ const invoke = await loadService();+ await invoke("transcription-start", "meeting");++ expect(h.worker.start).toHaveBeenCalledExactlyOnceWith({+ noteId: "meeting",+ modelDirectory: "/data/transcription-model",+ recordingDirectory: null+ });+ expect(h.sink.open).toHaveBeenCalledExactlyOnceWith(STARTED_AT);+ expect(h.worker.connectCapture).toHaveBeenCalledExactlyOnceWith("worker end");+ expect(h.capture.start).toHaveBeenCalledExactlyOnceWith("capture end");+ expect(h.sink.open.mock.invocationCallOrder[0])+ .toBeLessThan(h.capture.start.mock.invocationCallOrder[0]);+ });++ it("stores the utterances and tells every window which sources are pending", async () => {const invoke = await loadService();await invoke("transcription-start", "meeting");+ const utterance = { source: "system" as const, offsetMs: 0, text: "Hi" };++ h.listeners?.onUtterance(utterance);+ h.sink.write.mockImplementationOnce(() => {+ throw new Error("note deleted");+ });+ h.listeners?.onUtterance(utterance);+ expect(h.sink.write).toHaveBeenCalledTimes(2);+ expect(h.log.error).toHaveBeenCalledWith(+ "Transcription: saving an utterance failed: Error: note deleted");- h.sessionOptions?.onPendingChanged?.([ "system" ]);+ h.listeners?.onPendingChanged([ "system" ]);expect(h.broadcast.at(-1)).toEqual({status: "recording", noteId: "meeting", pendingSources: [ "system" ]});@@ -126,46 +160,66 @@ describe("transcription IPC", () => {});it("keeps the audio and the transcript as produced only in development builds", async () => {- const settings = { recognizer: { decodingMethod: "greedy_search" } };- h.loadEngine.mockResolvedValue({ settings });+ vi.useFakeTimers({ now: STARTED_AT, toFake: [ "Date" ] });const invoke = await loadService();await invoke("transcription-start", "meeting");- expect(h.sessionOptions?.sinks).toHaveLength(1);- expect(h.sessionOptions?.taps).toEqual([]);+ expect(h.worker.start.mock.calls[0][0].recordingDirectory).toBeNull();await invoke("transcription-stop");h.isDev = true;await invoke("transcription-start", "meeting");- expect(h.sessionOptions?.sinks[1]).toEqual({ directory: "/recordings/meeting" });- expect(h.sessionOptions?.taps).toEqual([- { directory: "/recordings/meeting", noteId: "meeting", settings }- ]);+ expect(h.worker.start.mock.calls[1][0].recordingDirectory)+ .toBe("/data/transcription-recordings/2026-10-14T09-30-00-000Z_meeting");+ vi.useRealTimers();});- it("refuses a note that does not exist, without changing state", async () => {+ it("refuses a missing note or model, without changing state", async () => {const invoke = await loadService();expect(await invoke("transcription-start", "deleted")).toBe("transcription.note-not-found");expect(await invoke("transcription-start", { noteId: "meeting" })).toBe("transcription.note-not-found");- expect(h.loadEngine).not.toHaveBeenCalled();+ h.isModelInstalled = false;+ expect(await invoke("transcription-start", "meeting"))+ .toBe(`transcription.model-missing {"directory":"/data/transcription-model"}`);+ expect(h.listeners).toBeNull();expect(h.broadcast).toEqual([]);});- it("explains why it could not start and returns to idle", async () => {+ it("explains why it could not start, ends the process and returns to idle", async () => {const invoke = await loadService();- h.loadEngine.mockRejectedValueOnce(new ModelMissingError("/data/transcription-model"));+ h.worker.start.mockRejectedValueOnce(new Error("Protobuf parsing failed"));expect(await invoke("transcription-start", "meeting"))- .toBe(`transcription.model-missing {"directory":"/data/transcription-model"}`);+ .toBe(`transcription.start-failed {"message":"Protobuf parsing failed"}`);+ expect(h.capture.start).not.toHaveBeenCalled();- h.sessionStart.mockRejectedValueOnce(new Error("No audio source could be captured."));+ h.capture.start.mockRejectedValueOnce(new Error("No audio source could be captured."));expect(await invoke("transcription-start", "meeting")).toBe(`transcription.start-failed {"message":"No audio source could be captured."}`);+ expect(h.worker.kill).toHaveBeenCalledTimes(2);+ expect(h.capture.stop).toHaveBeenCalledTimes(2);expect(await invoke("transcription-get-state")).toEqual(IDLE);await invoke("transcription-stop");- expect(h.sessionStop).not.toHaveBeenCalled();+ expect(h.worker.stop).not.toHaveBeenCalled();+ });++ it("ends a recording whose transcription process died, and says why once", async () => {+ const invoke = await loadService();+ await invoke("transcription-start", "meeting");++ h.listeners?.onUnexpectedExit(139);+ expect(h.capture.stop).toHaveBeenCalledOnce();+ expect(h.broadcast.at(-1))+ .toEqual({ ...IDLE, error: "transcription.stopped-unexpectedly" });+ expect(await invoke("transcription-get-state")).toEqual(IDLE);+ expect(h.log.error).toHaveBeenCalledWith(+ "Transcription: the transcription process exited with code 139.");++ await invoke("transcription-stop");+ expect(h.worker.stop).not.toHaveBeenCalled();+ expect(await invoke("transcription-start", "meeting")).toBe("");});});diff --git a/apps/desktop/src/services/transcription/index.ts b/apps/desktop/src/services/transcription/index.tsindex c6c77987c6..85dac744be 100644--- a/apps/desktop/src/services/transcription/index.ts+++ b/apps/desktop/src/services/transcription/index.ts@@ -1,18 +1,30 @@-import type { TranscriptionState } from "@triliumnext/commons";+import type { TranscriptionState, TranscriptUtterance } from "@triliumnext/commons";import { becca, getLog, utils as coreUtils } from "@triliumnext/core";-import { BrowserWindow, ipcMain } from "electron";+import dataDirs from "@triliumnext/server/src/services/data_dir.js";+import { BrowserWindow, ipcMain, MessageChannelMain } from "electron";+import fs from "fs";import { t } from "i18next";+import path from "path";import { AttachmentSink } from "./attachment_sink";-import { getRecordingDirectory, TranscriptFileSink, WavRecorder } from "./audio_recorder";-import { ElectronAudioCapture } from "./capture";-import { loadSherpaEngine, ModelMissingError, type SpeechEngine } from "./engine";-import { type AudioTap, RecordingSession, type TranscriptSink } from "./session";+import { CaptureWindow } from "./capture";+import { TranscriptionWorker } from "./worker_client";++/** Where the models live: Parakeet TDT 0.6B v3 (int8) plus `silero_vad.onnx`. */+export const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");++/** Where development builds keep the audio of each recording, one directory per recording. */+export const RECORDINGS_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-recordings");const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };+interface Recording {+ worker: TranscriptionWorker;+ capture: CaptureWindow;+}+let state = IDLE;-let activeSession: RecordingSession | null = null;+let activeRecording: Recording | null = null;export function setupTranscriptionHandlers() {ipcMain.handle("transcription-start", (_event, noteId: unknown) => startRecording(noteId));@@ -21,7 +33,8 @@ export function setupTranscriptionHandlers() {}/**- * Starts recording for the meeting note `noteId`.+ * Starts recording for the meeting note `noteId`. The engine runs in a utility process, which+ * the capture window posts its samples to; the main process only stores the utterances.** @returns An empty string once recording runs, or the reason it cannot start.*/@@ -32,41 +45,56 @@ export async function startRecording(noteId: unknown): Promise<string> {if (typeof noteId !== "string" || !becca.getNote(noteId)) {return t("transcription.note-not-found");}+ if (!fs.existsSync(MODEL_DIR)) {+ return t("transcription.model-missing", { directory: MODEL_DIR });+ }setState({ status: "starting", noteId, pendingSources: [] });+ const sink = new AttachmentSink(noteId);+ const recording: Recording = {+ capture: new CaptureWindow(),+ worker: new TranscriptionWorker({+ onUtterance: (utterance) => writeUtterance(sink, utterance),+ onPendingChanged: (pendingSources) => setState({ ...state, pendingSources }),+ onUnexpectedExit: (code) => handleUnexpectedExit(recording, code),+ log: getLog()+ })+ };+ activeRecording = recording;try {- const engine = await loadSherpaEngine();- const session = new RecordingSession({- capture: new ElectronAudioCapture(),- engine,- ...createReceivers(noteId, engine),- onPendingChanged: (pendingSources) => setState({ ...state, pendingSources })+ const { startedAt } = await recording.worker.start({+ noteId,+ modelDirectory: MODEL_DIR,+ recordingDirectory: coreUtils.isDev() ? getRecordingDirectory(Date.now(), noteId) : null});- const sources = await session.start();- activeSession = session;+ sink.open(startedAt);+ const { port1, port2 } = new MessageChannelMain();+ recording.worker.connectCapture(port2);+ const sources = await recording.capture.start(port1);getLog().info(`Transcription: recording ${sources.join(" and ")} for note ${noteId}.`);setState({ ...state, status: "recording" });return "";} catch (e) {+ activeRecording = null;+ recording.capture.stop();+ recording.worker.kill();setState(IDLE);- if (e instanceof ModelMissingError) {- return t("transcription.model-missing", { directory: e.directory });- }getLog().error(`Transcription did not start: ${(e as Error)?.stack ?? e}`);return t("transcription.start-failed", { message: (e as Error)?.message ?? String(e) });}}export async function stopRecording() {- const session = activeSession;- if (state.status !== "recording" || !session) {+ const recording = activeRecording;+ if (state.status !== "recording" || !recording) {return;}setState({ ...state, status: "stopping" });- activeSession = null;+ activeRecording = null;+ recording.capture.stop();try {- await session.stop();+ await recording.worker.stop();getLog().info(`Transcription: recording for note ${state.noteId} stopped.`);} finally {setState(IDLE);@@ -77,24 +105,42 @@ export function getState() {return state;}+/** The directory under {@link RECORDINGS_DIR} for a recording of `noteId` begun at `createdAt`. */+export function getRecordingDirectory(createdAt: number, noteId: string) {+ const timestamp = new Date(createdAt).toISOString().replace(/[:.]/g, "-");+ return path.join(RECORDINGS_DIR, `${timestamp}_${noteId}`);+}++function writeUtterance(sink: AttachmentSink, utterance: TranscriptUtterance) {+ try {+ sink.write(utterance);+ } catch (e) {+ getLog().error(`Transcription: saving an utterance failed: ${e}`);+ }+}+/**- * Where the utterances and the audio of a recording for `noteId` go. Development builds also keep- * the audio and the transcript as produced, to transcribe the audio again while the engine is tuned.+ * Ends a recording whose transcription process died, keeping what was transcribed, and tells+ * every window why it ended.*/-function createReceivers(noteId: string, engine: SpeechEngine) {- const sinks: TranscriptSink[] = [ new AttachmentSink(noteId) ];- const taps: AudioTap[] = [];- if (coreUtils.isDev()) {- const directory = getRecordingDirectory(Date.now(), noteId);- sinks.push(new TranscriptFileSink(directory));- taps.push(new WavRecorder(directory, noteId, engine.settings));+function handleUnexpectedExit(recording: Recording, code: number) {+ if (activeRecording !== recording) {+ return;}- return { sinks, taps };+ activeRecording = null;+ recording.capture.stop();+ getLog().error(`Transcription: the transcription process exited with code ${code}.`);+ state = IDLE;+ broadcast({ ...IDLE, error: t("transcription.stopped-unexpectedly") });}function setState(newState: TranscriptionState) {state = newState;+ broadcast(state);+}++function broadcast(stateToSend: TranscriptionState) {for (const window of BrowserWindow.getAllWindows()) {- window.webContents.send("transcription-state", state);+ window.webContents.send("transcription-state", stateToSend);}}diff --git a/apps/desktop/src/services/transcription/protocol.ts b/apps/desktop/src/services/transcription/protocol.tsnew file mode 100644index 0000000000..dfbb8e1d42--- /dev/null+++ b/apps/desktop/src/services/transcription/protocol.ts@@ -0,0 +1,40 @@+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";++/** Every source and engine works on mono 16 kHz samples. */+export const SAMPLE_RATE = 16000;++/** Where the transcription process reports to, since it has no access to Trilium's log. */+export interface Logger {+ info(message: string): void;+ error(message: string): void;+}++/** What the main process sends the transcription process, in this order. */+export type WorkerRequest =+ | {+ type: "start";+ noteId: string;+ modelDirectory: string;+ /** Where development builds keep the audio and the transcript as produced, else `null`. */+ recordingDirectory: string | null;+ }+ /** Sent once the recording runs, with the port the capture page posts its samples to. */+ | { type: "capture" }+ | { type: "stop" };++/** What the transcription process reports to the main process. */+export type WorkerEvent =+ /** The engine is loaded and the recording's start time is set; capturing can begin. */+ | { type: "ready"; startedAt: number; settings: Record<string, unknown> }+ | { type: "failed"; message: string }+ | { type: "pending"; sources: TranscriptSource[] }+ | { type: "utterance"; utterance: TranscriptUtterance }+ | { type: "log"; level: keyof Logger; message: string }+ /** Every utterance captured before `stop` has been sent. */+ | { type: "stopped" };++/** One chunk of samples, posted by the capture page straight to the transcription process. */+export interface CaptureChunk {+ source: TranscriptSource;+ samples: Float32Array;+}diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.tsindex 7b5545f74c..0dc605d1d8 100644--- a/apps/desktop/src/services/transcription/session.spec.ts+++ b/apps/desktop/src/services/transcription/session.spec.ts@@ -1,15 +1,12 @@import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";import { beforeEach, describe, expect, it, vi } from "vitest";-import type { SamplesListener } from "./capture";import type { SpeakerTracker, SpeechSegment } from "./engine";-import { type AudioTap, RecordingSession, type TranscriptSink } from "./session";+import {+ type AudioTap, RecordingSession, type SamplesListener, type TranscriptSink+} from "./session";-const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));-vi.mock("@triliumnext/core", () => ({ getLog: () => log }));-vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({- default: { TRILIUM_DATA_DIR: "/data" }-}));+const log = { info: vi.fn(), error: vi.fn() };/*** A chunk the fake segmenter reads as a finished utterance: `id` becomes the transcribed text@@ -36,9 +33,8 @@ function createHarness(extraSinks: TranscriptSink[] = [], taps: AudioTap[] = [])let clock = 1000;let onSamples: SamplesListener = () => {};const capture = {- start: vi.fn(async (listener: SamplesListener) => {+ start: vi.fn((listener: SamplesListener) => {onSamples = listener;- return [ "microphone", "system" ] as TranscriptSource[];}),stop: vi.fn()};@@ -77,7 +73,7 @@ function createHarness(extraSinks: TranscriptSink[] = [], taps: AudioTap[] = [])const sinks = [ ...extraSinks, sink ];const onPendingChanged = vi.fn();const session = new RecordingSession({- capture, engine, sinks, taps, onPendingChanged, now: () => clock+ capture, engine, sinks, taps, log, onPendingChanged, now: () => clock});return {@@ -99,6 +95,7 @@ describe("RecordingSession", () => {const { session, capture, engine, sink, deliver, written } = createHarness();await session.start();expect(sink.open).toHaveBeenCalledExactlyOnceWith(1000);+ expect(session.startedAt).toBe(1000);expect(sink.open.mock.invocationCallOrder[0]).toBeLessThan(capture.start.mock.invocationCallOrder[0]);@@ -237,7 +234,7 @@ describe("RecordingSession", () => {"Transcription: identifying the speaker on system failed: Error: model crashed");});- it("logs a failing source instead of throwing into the IPC listener", async () => {+ it("logs a failing source instead of throwing into the capture", async () => {const { session, engine, deliver } = createHarness();engine.createSegmenter.mockImplementationOnce(() => {throw new Error("VAD model missing");diff --git a/apps/desktop/src/services/transcription/session.ts b/apps/desktop/src/services/transcription/session.tsindex e733b6c9b2..2019e0a87d 100644--- a/apps/desktop/src/services/transcription/session.ts+++ b/apps/desktop/src/services/transcription/session.ts@@ -1,14 +1,20 @@import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";-import { getLog } from "@triliumnext/core";-import type { AudioCapture } from "./capture";-import {- SAMPLE_RATE, type SpeakerTracker, type SpeechEngine, type SpeechSegment, type SpeechSegmenter-} from "./engine";+import type { SpeakerTracker, SpeechEngine, SpeechSegment, SpeechSegmenter } from "./engine";+import { type Logger, SAMPLE_RATE } from "./protocol";/** The sources whose speakers are told apart; the microphone carries only the local speaker. */const DIARIZED_SOURCES: readonly TranscriptSource[] = [ "system" ];+export type SamplesListener = (source: TranscriptSource, samples: Float32Array) => void;++/** Produces mono {@link SAMPLE_RATE} samples per source until stopped. */+export interface AudioCapture {+ /** Starts delivering samples to `onSamples`. */+ start(onSamples: SamplesListener): void;+ stop(): void;+}+/** Receives the utterances of a recording in the order they were spoken per source. */export interface TranscriptSink {/**@@ -39,6 +45,7 @@ export interface RecordingSessionOptions {engine: SpeechEngine;sinks: TranscriptSink[];taps?: AudioTap[];+ log: Logger;/*** Called with the sources whose speech has not reached the sinks yet, either because it is* still being spoken or because it waits to be transcribed, every time that list changes.@@ -58,25 +65,30 @@ export class RecordingSession {/** How many utterances of each source wait to be transcribed or written. */private readonly queued = new Map<TranscriptSource, number>();private queue = Promise.resolve();- private startedAt = 0;+ private startedAtMs = 0;private pendingSources: TranscriptSource[] = [];constructor(private readonly options: RecordingSessionOptions) {}+ /** When the recording started, in milliseconds since the epoch; set by {@link start}. */+ get startedAt() {+ return this.startedAtMs;+ }+async start() {- this.startedAt = this.now();+ this.startedAtMs = this.now();for (const receiver of [ ...this.options.sinks, ...this.options.taps ?? [] ]) {- await receiver.open?.(this.startedAt);+ await receiver.open?.(this.startedAtMs);}- return await this.options.capture.start((source, samples) => {- // An exception thrown from an `ipcMain` listener takes down the main process.+ this.options.capture.start((source, samples) => {+ // An exception thrown from the capture's listener ends the transcription process.try {const pipeline = this.pipelineFor(source, samples);this.tap(source, samples, pipeline.firstSampleOffsetMs);pipeline.accept(samples);this.updatePending();} catch (e) {- getLog().error(`Transcription: ${source} failed: ${(e as Error)?.message ?? e}`);+ this.log.error(`Transcription: ${source} failed: ${(e as Error)?.message ?? e}`);}});}@@ -92,7 +104,7 @@ export class RecordingSession {try {await tap.close?.();} catch (e) {- getLog().error(`Transcription: closing an audio tap failed: ${e}`);+ this.log.error(`Transcription: closing an audio tap failed: ${e}`);}}await this.queue;@@ -101,7 +113,7 @@ export class RecordingSession {try {await sink.close?.();} catch (e) {- getLog().error(`Transcription: closing a sink failed: ${e}`);+ this.log.error(`Transcription: closing a sink failed: ${e}`);}}}@@ -111,7 +123,7 @@ export class RecordingSession {try {tap.write(source, samples, firstSampleOffsetMs);} catch (e) {- getLog().error(`Transcription: an audio tap failed on ${source}: ${e}`);+ this.log.error(`Transcription: an audio tap failed on ${source}: ${e}`);}}}@@ -124,8 +136,9 @@ export class RecordingSession {pipeline = new SourcePipeline(source,this.options.engine.createSegmenter(),- Math.max(0, this.now() - this.startedAt - chunkMs),- (offsetMs, samples) => this.enqueue(source, offsetMs, samples)+ Math.max(0, this.now() - this.startedAtMs - chunkMs),+ (offsetMs, samples) => this.enqueue(source, offsetMs, samples),+ this.options.log);this.pipelines.set(source, pipeline);const tracker = DIARIZED_SOURCES.includes(source)@@ -155,21 +168,21 @@ export class RecordingSession {try {await sink.write(utterance);} catch (e) {- getLog().error(`Transcription: a sink failed: ${e}`);+ this.log.error(`Transcription: a sink failed: ${e}`);}}- }).catch((e) => getLog().error(`Transcription failed: ${e}`)).finally(() => {+ }).catch((e) => this.log.error(`Transcription failed: ${e}`)).finally(() => {this.queued.set(source, (this.queued.get(source) ?? 1) - 1);this.updatePending();});}- /** Runs on the main thread, synchronously: about 100 ms per utterance. */+ /** Runs synchronously, about 100 ms per utterance, so the next chunks wait meanwhile. */private identifySpeaker(source: TranscriptSource, samples: Float32Array) {try {return this.speakerTrackers.get(source)?.identify(samples) ?? null;} catch (e) {- getLog().error(`Transcription: identifying the speaker on ${source} failed: ${e}`);+ this.log.error(`Transcription: identifying the speaker on ${source} failed: ${e}`);return null;}}@@ -189,6 +202,10 @@ export class RecordingSession {private now() {return (this.options.now ?? Date.now)();}++ private get log() {+ return this.options.log;+ }}/** Segments one source and logs its peak level every 5 s, to spot a silent source. */@@ -201,7 +218,8 @@ class SourcePipeline {private readonly segmenter: SpeechSegmenter,/** Milliseconds from the start of the recording to the source's first sample. */readonly firstSampleOffsetMs: number,- private readonly onSegment: (offsetMs: number, samples: Float32Array) => void+ private readonly onSegment: (offsetMs: number, samples: Float32Array) => void,+ private readonly log: Logger) {}accept(samples: Float32Array) {@@ -230,7 +248,7 @@ class SourcePipeline {}this.peakSampleCount += samples.length;if (this.peakSampleCount >= 5 * SAMPLE_RATE) {- getLog().info(`Transcription: ${this.source} peak ${this.peak.toFixed(4)} over 5 s`);+ this.log.info(`Transcription: ${this.source} peak ${this.peak.toFixed(4)} over 5 s`);this.peak = 0;this.peakSampleCount = 0;}diff --git a/apps/desktop/src/services/transcription/transcription_worker.ts b/apps/desktop/src/services/transcription/transcription_worker.tsnew file mode 100644index 0000000000..abe6bfbea5--- /dev/null+++ b/apps/desktop/src/services/transcription/transcription_worker.ts@@ -0,0 +1,113 @@+/**+ * The transcription process: an Electron utility process that runs one recording's VAD, speaker+ * embeddings and recognizer, so none of them takes time from the main process. The capture page+ * posts its samples here directly; the main process only receives the utterances. Started by+ * `TranscriptionWorker` in `worker_client.ts`, which also ends it.+ */+import type { TranscriptSource } from "@triliumnext/commons";+import type { MessagePortMain } from "electron";++import { TranscriptFileSink, WavRecorder } from "./audio_recorder";+import { loadSherpaEngine } from "./engine";+import type { CaptureChunk, Logger, WorkerEvent, WorkerRequest } from "./protocol";+import {+ type AudioCapture, type AudioTap, RecordingSession, type SamplesListener, type TranscriptSink+} from "./session";++const log: Logger = {+ info: (message) => post({ type: "log", level: "info", message }),+ error: (message) => post({ type: "log", level: "error", message })+};++type StartRequest = Extract<WorkerRequest, { type: "start" }>;++let session: RecordingSession | null = null;+let capture: PortCapture | null = null;++process.parentPort.on("message", ({ data, ports }) => {+ const request = data as WorkerRequest;+ switch (request.type) {+ case "start":+ start(request).catch((e) => {+ post({ type: "failed", message: (e as Error)?.message ?? String(e) });+ });+ break;+ case "capture":+ if (ports[0]) {+ capture?.connect(ports[0]);+ }+ break;+ case "stop":+ void stop();+ break;+ }+});++async function start({ noteId, modelDirectory, recordingDirectory }: StartRequest) {+ const engine = await loadSherpaEngine(modelDirectory);+ const sinks: TranscriptSink[] = [+ { write: (utterance) => post({ type: "utterance", utterance }) }+ ];+ const taps: AudioTap[] = [];+ if (recordingDirectory) {+ sinks.push(new TranscriptFileSink(recordingDirectory));+ taps.push(new WavRecorder(recordingDirectory, noteId, engine.settings, log));+ }++ capture = new PortCapture();+ const newSession = new RecordingSession({+ capture,+ engine,+ sinks,+ taps,+ log,+ onPendingChanged: (sources) => post({ type: "pending", sources })+ });+ await newSession.start();+ session = newSession;+ post({ type: "ready", startedAt: newSession.startedAt, settings: engine.settings });+}++async function stop() {+ try {+ await session?.stop();+ } catch (e) {+ log.error(`Transcription: stopping failed: ${e}`);+ }+ post({ type: "stopped" });+}++function post(event: WorkerEvent) {+ process.parentPort.postMessage(event);+}++/** The samples the capture page posts to the port the main process connects. */+class PortCapture implements AudioCapture {+ private port: MessagePortMain | null = null;+ private onSamples: SamplesListener | null = null;++ start(onSamples: SamplesListener) {+ this.onSamples = onSamples;+ }++ connect(port: MessagePortMain) {+ this.port = port;+ port.on("message", ({ data }) => {+ const chunk = data as CaptureChunk | null;+ if (chunk && isSource(chunk.source) && chunk.samples instanceof Float32Array) {+ this.onSamples?.(chunk.source, chunk.samples);+ }+ });+ port.start();+ }++ stop() {+ this.onSamples = null;+ this.port?.close();+ this.port = null;+ }+}++function isSource(source: unknown): source is TranscriptSource {+ return source === "microphone" || source === "system";+}diff --git a/apps/desktop/src/services/transcription/worker_client.spec.ts b/apps/desktop/src/services/transcription/worker_client.spec.tsnew file mode 100644index 0000000000..edabc231da--- /dev/null+++ b/apps/desktop/src/services/transcription/worker_client.spec.ts@@ -0,0 +1,140 @@+import { EventEmitter } from "events";+import path from "path";+import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from "vitest";++import type { Logger, WorkerEvent } from "./protocol";+import { TranscriptionWorker, type WorkerListeners } from "./worker_client";++const h = vi.hoisted(() => ({+ fork: vi.fn(),+ hasBundle: false+}));++vi.mock("electron", () => ({ utilityProcess: { fork: h.fork } }));+vi.mock("fs", () => ({+ default: { existsSync: (file: string) => !file.endsWith(".cjs") || h.hasBundle }+}));++class FakeProcess extends EventEmitter {+ postMessage = vi.fn();+ kill = vi.fn(() => {+ this.emit("exit", 0);+ return true;+ });++ report(event: WorkerEvent) {+ this.emit("message", event);+ }+}++let child: FakeProcess;+let listeners: {+ [ name in keyof Omit<WorkerListeners, "log"> ]: Mock<WorkerListeners[name]>+} & { log: { info: Mock<Logger["info"]>; error: Mock<Logger["error"]> } };++beforeEach(() => {+ vi.clearAllMocks();+ h.hasBundle = false;+ child = new FakeProcess();+ h.fork.mockReturnValue(child);+ listeners = {+ onUtterance: vi.fn(),+ onPendingChanged: vi.fn(),+ onUnexpectedExit: vi.fn(),+ log: { info: vi.fn(), error: vi.fn() }+ };+});++afterEach(() => {+ vi.useRealTimers();+});++const REQUEST = { noteId: "meeting", modelDirectory: "/models", recordingDirectory: null };++describe("TranscriptionWorker", () => {+ it("starts the bundled worker, or the sources when there is no bundle", () => {+ new TranscriptionWorker(listeners);+ h.hasBundle = true;+ new TranscriptionWorker(listeners);++ expect(h.fork.mock.calls.map(([ file ]) => path.basename(file as string)))+ .toEqual([ "transcription_worker.ts", "transcription_worker.cjs" ]);+ });++ it("resolves once the engine is ready, then relays what the process reports", async () => {+ const worker = new TranscriptionWorker(listeners);+ const started = worker.start(REQUEST);+ expect(child.postMessage).toHaveBeenCalledExactlyOnceWith({ type: "start", ...REQUEST });++ child.report({ type: "ready", startedAt: 1000, settings: { vad: {} } });+ await expect(started).resolves.toEqual({+ type: "ready", startedAt: 1000, settings: { vad: {} }+ });++ const port = { start: vi.fn() };+ worker.connectCapture(port as never);+ expect(child.postMessage).toHaveBeenLastCalledWith({ type: "capture" }, [ port ]);++ const utterance = { source: "system" as const, offsetMs: 0, text: "Hi" };+ child.report({ type: "utterance", utterance });+ child.report({ type: "pending", sources: [ "system" ] });+ child.report({ type: "log", level: "error", message: "system failed" });+ expect(listeners.onUtterance).toHaveBeenCalledExactlyOnceWith(utterance);+ expect(listeners.onPendingChanged).toHaveBeenCalledExactlyOnceWith([ "system" ]);+ expect(listeners.log.error).toHaveBeenCalledExactlyOnceWith("system failed");+ });++ it("rejects the start when the engine fails or the process dies first", async () => {+ const failing = new TranscriptionWorker(listeners).start(REQUEST);+ child.report({ type: "failed", message: "Protobuf parsing failed" });+ await expect(failing).rejects.toThrow("Protobuf parsing failed");++ child = new FakeProcess();+ h.fork.mockReturnValue(child);+ const crashing = new TranscriptionWorker(listeners).start(REQUEST);+ child.emit("exit", 139);+ await expect(crashing).rejects.toThrow("The transcription process exited with code 139.");+ expect(listeners.onUnexpectedExit).not.toHaveBeenCalled();+ });++ it("stops once the process has reported everything, then ends it", async () => {+ const worker = new TranscriptionWorker(listeners);+ let isStopped = false;+ const stopped = worker.stop().then(() => {+ isStopped = true;+ });+ expect(child.postMessage).toHaveBeenCalledExactlyOnceWith({ type: "stop" });+ await Promise.resolve();+ expect(isStopped).toBe(false);++ child.report({ type: "stopped" });+ await stopped;+ expect(child.kill).toHaveBeenCalledOnce();+ expect(listeners.onUnexpectedExit).not.toHaveBeenCalled();+ await worker.stop();+ expect(child.postMessage).toHaveBeenCalledOnce();+ });++ it("ends a process that does not stop in time", async () => {+ vi.useFakeTimers();+ const worker = new TranscriptionWorker(listeners);+ const stopped = worker.stop();++ await vi.advanceTimersByTimeAsync(2 * 60 * 1000);+ await stopped;+ expect(child.kill).toHaveBeenCalledOnce();+ expect(listeners.log.error).toHaveBeenCalledExactlyOnceWith(+ "Transcription: the transcription process did not stop in time.");+ });++ it("reports a process that exits while recording, but not one it ended", () => {+ new TranscriptionWorker(listeners);+ child.emit("exit", 139);+ expect(listeners.onUnexpectedExit).toHaveBeenCalledExactlyOnceWith(139);++ child = new FakeProcess();+ h.fork.mockReturnValue(child);+ new TranscriptionWorker(listeners).kill();+ expect(listeners.onUnexpectedExit).toHaveBeenCalledOnce();+ });+});diff --git a/apps/desktop/src/services/transcription/worker_client.ts b/apps/desktop/src/services/transcription/worker_client.tsnew file mode 100644index 0000000000..284d63b12e--- /dev/null+++ b/apps/desktop/src/services/transcription/worker_client.ts@@ -0,0 +1,129 @@+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";+import { type MessagePortMain, utilityProcess, type UtilityProcess } from "electron";+import fs from "fs";+import path from "path";++import type { Logger, WorkerEvent, WorkerRequest } from "./protocol";++export interface WorkerListeners {+ onUtterance(utterance: TranscriptUtterance): void;+ onPendingChanged(sources: TranscriptSource[]): void;+ /** Called when the process ends without having been stopped, as when it crashes. */+ onUnexpectedExit(code: number): void;+ log: Logger;+}++type ReadyEvent = Extract<WorkerEvent, { type: "ready" }>;+type StartRequest = Extract<WorkerRequest, { type: "start" }>;++/** How long {@link TranscriptionWorker.stop} waits for the speech still queued. */+const STOP_TIMEOUT_MS = 2 * 60 * 1000;++/**+ * The transcription process (`transcription_worker.ts`), from the main process: starts it,+ * connects the capture page to it and relays what it reports. One instance runs one recording;+ * the process ends with it.+ */+export class TranscriptionWorker {+ private readonly child: UtilityProcess;+ private startRequest: { resolve(event: ReadyEvent): void; reject(error: Error): void } | null =+ null;+ private isStopping = false;+ private hasExited = false;++ constructor(private readonly listeners: WorkerListeners) {+ this.child = utilityProcess.fork(getWorkerEntry(), [], {+ serviceName: "Trilium transcription"+ });+ this.child.on("message", (event: WorkerEvent) => this.handle(event));+ this.child.on("exit", (code) => this.handleExit(code));+ }++ /** Loads the engine and starts the recording, without capturing yet. */+ start(request: Omit<StartRequest, "type">) {+ return new Promise<ReadyEvent>((resolve, reject) => {+ this.startRequest = { resolve, reject };+ this.child.postMessage({ type: "start", ...request } satisfies WorkerRequest);+ });+ }++ /** Hands over the port the capture page posts its samples to. */+ connectCapture(port: MessagePortMain) {+ this.child.postMessage({ type: "capture" } satisfies WorkerRequest, [ port ]);+ }++ /**+ * Waits until the speech already captured has been reported, then ends the process. Ends it+ * regardless after {@link STOP_TIMEOUT_MS}.+ */+ async stop() {+ if (this.hasExited) {+ return;+ }+ this.isStopping = true;+ const exited = new Promise<void>((resolve) => this.child.once("exit", () => resolve()));+ const timeout = setTimeout(() => {+ this.listeners.log.error(+ "Transcription: the transcription process did not stop in time.");+ this.child.kill();+ }, STOP_TIMEOUT_MS);+ this.child.postMessage({ type: "stop" } satisfies WorkerRequest);+ await exited;+ clearTimeout(timeout);+ }++ /** Ends the process at once, dropping what it has not reported. */+ kill() {+ this.isStopping = true;+ this.child.kill();+ }++ private handle(event: WorkerEvent) {+ switch (event.type) {+ case "ready":+ this.startRequest?.resolve(event);+ this.startRequest = null;+ break;+ case "failed":+ this.startRequest?.reject(new Error(event.message));+ this.startRequest = null;+ break;+ case "pending":+ this.listeners.onPendingChanged(event.sources);+ break;+ case "utterance":+ this.listeners.onUtterance(event.utterance);+ break;+ case "log":+ this.listeners.log[event.level](event.message);+ break;+ case "stopped":+ this.child.kill();+ break;+ }+ }++ private handleExit(code: number) {+ this.hasExited = true;+ const error = new Error(`The transcription process exited with code ${code}.`);+ if (this.startRequest) {+ this.startRequest.reject(error);+ this.startRequest = null;+ } else if (!this.isStopping) {+ this.listeners.onUnexpectedExit(code);+ }+ }+}++/**+ * The bundled `transcription_worker.cjs` beside the main bundle, or the TypeScript when running+ * from sources, which the utility process reads through the `--import tsx` it inherits in+ * `NODE_OPTIONS`.+ */+function getWorkerEntry() {+ const candidates = [+ path.join(__dirname, "transcription_worker.cjs"),+ path.join(__dirname, "transcription_worker.ts")+ ];+ return candidates.find((candidate) => fs.existsSync(candidate)) ?? candidates[0];+}diff --git a/apps/server/src/assets/translations/en/server.json b/apps/server/src/assets/translations/en/server.jsonindex 20f20c845b..0bfb215580 100644--- a/apps/server/src/assets/translations/en/server.json+++ b/apps/server/src/assets/translations/en/server.json@@ -567,6 +567,7 @@"note-not-found": "The meeting note no longer exists.","model-missing": "No speech recognition model was found in {{- directory}}.","start-failed": "The transcription could not start: {{- message}}",+ "stopped-unexpectedly": "The transcription stopped unexpectedly. What was transcribed until then is kept.","attachment-title": "Transcript"}}diff --git a/packages/commons/src/lib/electron_api_interface.ts b/packages/commons/src/lib/electron_api_interface.tsindex 80e6fad3af..7b762fa23b 100644--- a/packages/commons/src/lib/electron_api_interface.ts+++ b/packages/commons/src/lib/electron_api_interface.ts@@ -724,6 +724,11 @@ export interface TranscriptionState {noteId: string | null;/** The sources heard speaking, or whose speech is still being transcribed. */pendingSources: TranscriptSource[];+ /**+ * Why the recording ended without being stopped, sent once with the `idle` state that+ * follows; `getState()` never returns it.+ */+ error?: string;}/**diff --git a/scripts/build-utils.ts b/scripts/build-utils.tsindex 027970b582..f3ae6f2833 100644--- a/scripts/build-utils.ts+++ b/scripts/build-utils.ts@@ -92,7 +92,9 @@ export default class BuildHelper {// The Codex ACP adapter is copied to RESOURCE_DIR/codex-acp.mjs by the// build; codex_binary.ts only require.resolve()s the package when running// from source.- "@agentclientprotocol/codex-acp"+ "@agentclientprotocol/codex-acp",+ // Loads its native addon from its own platform package at runtime.+ "sherpa-onnx-node"],metafile: true,loader: {