Skip to content

Build your personal knowledge base with Trilium Notes

feat(desktop): transcribe meetings in a utility process

Elian Doran8e9ef5d170be
diff --git a/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsx b/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsx
index fb334d388b..95f6aa8a95 100644
--- a/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsx
+++ b/apps/client/src/widgets/launch_bar/RecordMeetingButton.spec.tsx
@@ -1,5 +1,10 @@
+import type { TranscriptionState } from "@triliumnext/commons";
+import { act } from "preact/test-utils";
import { beforeEach, describe, expect, it, vi } from "vitest";
+import type FNote from "../../entities/fnote";
+import { renderInto } from "../../test/render";
+
const mocks = vi.hoisted(() => ({
getInboxNote: vi.fn(),
createNote: vi.fn(),
@@ -18,8 +23,12 @@ vi.mock("../../services/toast", () => ({
vi.mock("../../services/i18n", () => ({
t: (key: string, options?: { date?: string }) => `${key} ${options?.date}`
}));
+vi.mock("./launch_bar_widgets", () => ({
+ LaunchBarActionButton: () => null,
+ useLauncherIconAndTitle: () => ({ icon: "bx bx-microphone", title: "Record meeting" })
+}));
-import { createMeetingNote, toggleRecording } from "./RecordMeetingButton";
+import RecordMeetingButton, { createMeetingNote, toggleRecording } from "./RecordMeetingButton";
beforeEach(() => {
vi.clearAllMocks();
@@ -88,3 +97,34 @@ describe("toggleRecording", () => {
expect(mocks.createNote).not.toHaveBeenCalled();
});
});
+
+describe("RecordMeetingButton", () => {
+ it("says once why a recording stopped by itself", async () => {
+ let onStateChanged: (state: TranscriptionState) => void = () => {};
+ const report = (state: Partial<TranscriptionState>) => act(() => onStateChanged({
+ status: "idle", noteId: null, pendingSources: [], ...state
+ }));
+ window.electronApi = {
+ transcription: {
+ getState: async () => ({ status: "idle", noteId: null, pendingSources: [] }),
+ onStateChanged: (callback: typeof onStateChanged) => {
+ onStateChanged = callback;
+ return () => {};
+ }
+ }
+ } as unknown as typeof window.electronApi;
+ act(() => {
+ renderInto(<RecordMeetingButton launcherNote={{} as FNote} />);
+ });
+
+ await report({ status: "recording", noteId: "meeting" });
+ await report({ error: "Stopped unexpectedly" });
+ await report({ error: "Stopped unexpectedly", pendingSources: [] });
+ expect(mocks.showError).toHaveBeenCalledExactlyOnceWith("Stopped unexpectedly");
+
+ await report({ status: "recording", noteId: "meeting" });
+ await report({ error: "Stopped unexpectedly" });
+ expect(mocks.showError).toHaveBeenCalledTimes(2);
+ window.electronApi = undefined;
+ });
+});
diff --git a/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx b/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx
index 90dfa39808..34b18e7fd7 100644
--- a/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx
+++ b/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx
@@ -21,11 +21,17 @@ import {
*/
export default function RecordMeetingButton({ launcherNote }: LauncherNoteProps) {
const { icon, title } = useLauncherIconAndTitle(launcherNote);
- const { status } = useTranscriptionState();
+ const { status, error } = useTranscriptionState();
const handleClick = useCallback(() => {
void toggleRecording(status);
}, [ status ]);
+ useEffect(() => {
+ if (error) {
+ toast.showError(error);
+ }
+ }, [ error ]);
+
return (
<LaunchBarActionButton
launcherNote={launcherNote}
diff --git a/apps/desktop/scripts/build.ts b/apps/desktop/scripts/build.ts
index ffc2202a7a..a03ed0dfdb 100644
--- a/apps/desktop/scripts/build.ts
+++ b/apps/desktop/scripts/build.ts
@@ -20,6 +20,9 @@ async function main() {
// The image compression worker, which lives in the server it embeds. Built here too so the
// desktop app can compress off-thread rather than falling back to doing it in the main process.
await build.buildBackend([ "../server/src/services/image_worker.ts" ]);
+ // The meeting transcription process, which transcription/worker_client.ts starts by its .cjs
+ // path as an Electron utility process.
+ await build.buildBackend([ "src/services/transcription/transcription_worker.ts" ]);
await build.buildTesseractWorker("../server/src/services/ocr/tesseract_worker.ts");
// Copy assets.
diff --git a/apps/desktop/src/services/transcription/audio_recorder.spec.ts b/apps/desktop/src/services/transcription/audio_recorder.spec.ts
index 907d094027..5b2a401c7a 100644
--- a/apps/desktop/src/services/transcription/audio_recorder.spec.ts
+++ b/apps/desktop/src/services/transcription/audio_recorder.spec.ts
@@ -4,15 +4,10 @@ import path from "path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
- encodeWavHeader, getRecordingDirectory, type RecordingManifest, toPcm16, TranscriptFileSink,
- WavRecorder
+ encodeWavHeader, type RecordingManifest, toPcm16, TranscriptFileSink, WavRecorder
} from "./audio_recorder";
-const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));
-vi.mock("@triliumnext/core", () => ({ getLog: () => log }));
-vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({
- default: { TRILIUM_DATA_DIR: "/data" }
-}));
+const log = { info: vi.fn(), error: vi.fn() };
const STARTED_AT = Date.parse("2026-10-14T09:30:00.000Z");
const SETTINGS = { recognizer: { decodingMethod: "greedy_search" } };
@@ -35,16 +30,9 @@ function readJson<T>(fileName: string) {
return JSON.parse(fs.readFileSync(path.join(directory, fileName), "utf8")) as T;
}
-it("names a recording's directory after when it was created and its note", () => {
- expect(path.basename(getRecordingDirectory(STARTED_AT, "meeting")))
- .toBe("2026-10-14T09-30-00-000Z_meeting");
- expect(path.dirname(getRecordingDirectory(STARTED_AT, "meeting")))
- .toBe(path.join("/data", "transcription-recordings"));
-});
-
describe("WavRecorder", () => {
it("keeps each source in a WAV file, lined up by the manifest", () => {
- const recorder = new WavRecorder(directory, "meeting", SETTINGS);
+ const recorder = new WavRecorder(directory, "meeting", SETTINGS, log);
recorder.open(STARTED_AT);
recorder.write("microphone", new Float32Array([ 0.5, -0.5 ]), 1500);
recorder.write("system", new Float32Array([ 1 ]), 3000);
@@ -70,7 +58,7 @@ describe("WavRecorder", () => {
});
it("stops keeping the audio after a failed write, logging it once", () => {
- const recorder = new WavRecorder(directory, "meeting", SETTINGS);
+ const recorder = new WavRecorder(directory, "meeting", SETTINGS, log);
recorder.open(STARTED_AT);
fs.rmSync(directory, { recursive: true });
diff --git a/apps/desktop/src/services/transcription/audio_recorder.ts b/apps/desktop/src/services/transcription/audio_recorder.ts
index ea4de9b2bd..992c8304cd 100644
--- a/apps/desktop/src/services/transcription/audio_recorder.ts
+++ b/apps/desktop/src/services/transcription/audio_recorder.ts
@@ -1,24 +1,13 @@
import type { Transcript, TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
-import { getLog } from "@triliumnext/core";
-import dataDirs from "@triliumnext/server/src/services/data_dir.js";
import fs from "fs";
import path from "path";
-import { SAMPLE_RATE } from "./engine";
+import { type Logger, SAMPLE_RATE } from "./protocol";
import type { AudioTap, TranscriptSink } from "./session";
-/** Where development builds keep the audio of each recording, one directory per recording. */
-export const RECORDINGS_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-recordings");
-
/** The transcript the recording produced live, beside the audio it came from. */
export const TRANSCRIPT_FILE = "transcript.json";
-/** The directory under {@link RECORDINGS_DIR} for a recording of `noteId` created at `createdAt`. */
-export function getRecordingDirectory(createdAt: number, noteId: string) {
- const timestamp = new Date(createdAt).toISOString().replace(/[:.]/g, "-");
- return path.join(RECORDINGS_DIR, `${timestamp}_${noteId}`);
-}
-
/** The `recording.json` next to the audio files, which lines the sources up with the transcript. */
export interface RecordingManifest {
noteId: string;
@@ -47,7 +36,8 @@ export class WavRecorder implements AudioTap {
constructor(
private readonly directory: string,
private readonly noteId: string,
- private readonly engineSettings: Record<string, unknown>
+ private readonly engineSettings: Record<string, unknown>,
+ private readonly log: Logger
) {}
open(startedAt: number) {
@@ -60,7 +50,7 @@ export class WavRecorder implements AudioTap {
sources: {}
};
this.writeManifest();
- getLog().info(`Transcription: keeping the audio in ${this.directory}`);
+ this.log.info(`Transcription: keeping the audio in ${this.directory}`);
}
write(source: TranscriptSource, samples: Float32Array, firstSampleOffsetMs: number) {
@@ -82,7 +72,7 @@ export class WavRecorder implements AudioTap {
} catch (e) {
// Stops for good, so a full disk logs once rather than on every chunk.
this.failed = true;
- getLog().error(`Transcription: stopped keeping the audio: ${e}`);
+ this.log.error(`Transcription: stopped keeping the audio: ${e}`);
}
}
diff --git a/apps/desktop/src/services/transcription/capture.ts b/apps/desktop/src/services/transcription/capture.ts
index 50a4100d5e..b7442c2c34 100644
--- a/apps/desktop/src/services/transcription/capture.ts
+++ b/apps/desktop/src/services/transcription/capture.ts
@@ -1,27 +1,14 @@
import type { TranscriptSource } from "@triliumnext/commons";
import { getLog } from "@triliumnext/core";
-import { app, BrowserWindow, ipcMain, type IpcMainEvent, session } from "electron";
+import { app, BrowserWindow, type MessagePortMain, session } from "electron";
import fs from "fs";
import path from "path";
-import { SAMPLE_RATE } from "./engine";
+import { SAMPLE_RATE } from "./protocol";
-export type SamplesListener = (source: TranscriptSource, samples: Float32Array) => void;
+const PORT_CHANNEL = "transcription-capture-port";
-/** Produces mono {@link SAMPLE_RATE} samples per source until stopped. */
-export interface AudioCapture {
- /**
- * Starts every source it can and returns the ones that run.
- *
- * @throws when no source could start.
- */
- start(onSamples: SamplesListener): Promise<TranscriptSource[]>;
- stop(): void;
-}
-
-const SAMPLES_CHANNEL = "transcription-samples";
-
-/** Samples per IPC message. `VadSegmenter` drops what does not fill a 512-sample window. */
+/** Samples per message. `VadSegmenter` drops what does not fill a 512-sample window. */
const CHUNK_SAMPLES = 4096;
/** The page function that starts each source. */
@@ -30,27 +17,23 @@ const SOURCE_STARTERS = [
[ "system", "captureSystemAudio()" ]
] as const;
-type SamplesIpcListener =
- (event: IpcMainEvent, source: TranscriptSource, samples: Float32Array) => void;
-
/**
- * Captures the default microphone and the system audio output in a hidden window, which streams
- * the samples to the main process over IPC.
+ * Captures the default microphone and the system audio output in a hidden window, which posts
+ * them to a port as `CaptureChunk`s of {@link CHUNK_SAMPLES} mono samples. The samples never
+ * pass through the main process.
*/
-export class ElectronAudioCapture implements AudioCapture {
+export class CaptureWindow {
private window: BrowserWindow | null = null;
- private listener: SamplesIpcListener | null = null;
- async start(onSamples: SamplesListener) {
+ /**
+ * Starts every source it can, posting to `port`, and returns the ones that run.
+ *
+ * @throws when no source could start.
+ */
+ async start(port: MessagePortMain) {
const window = await createCaptureWindow();
this.window = window;
- this.listener = (event, source, samples) => {
- const isSource = source === "microphone" || source === "system";
- if (event.sender === window.webContents && isSource) {
- onSamples(source, samples);
- }
- };
- ipcMain.on(SAMPLES_CHANNEL, this.listener);
+ window.webContents.postMessage(PORT_CHANNEL, null, [ port ]);
const sources: TranscriptSource[] = [];
for (const [ source, call ] of SOURCE_STARTERS) {
@@ -74,10 +57,6 @@ export class ElectronAudioCapture implements AudioCapture {
}
stop() {
- if (this.listener) {
- ipcMain.removeListener(SAMPLES_CHANNEL, this.listener);
- this.listener = null;
- }
if (this.window && !this.window.isDestroyed()) {
this.window.destroy();
}
@@ -87,8 +66,9 @@ export class ElectronAudioCapture implements AudioCapture {
/**
* The window runs only {@link CAPTURE_PAGE}, a local file, so it gets Node integration to reach
- * `ipcRenderer` without a preload. Its in-memory session grants the microphone and screen capture
- * and nothing else; every screen capture request gets the window itself plus the system audio.
+ * `ipcRenderer`, which delivers the port, without a preload. Its in-memory session grants the
+ * microphone and screen capture and nothing else; every screen capture request gets the window
+ * itself plus the system audio.
*/
async function createCaptureWindow() {
const captureSession = session.fromPartition("transcription");
@@ -163,12 +143,14 @@ registerProcessor("chunk-collector", ChunkCollector);
const CAPTURE_PAGE = `<!DOCTYPE html>
<script>
const { ipcRenderer } = require("electron");
+const portReceived = new Promise((resolve) =>
+ ipcRenderer.once("${PORT_CHANNEL}", (e) => resolve(e.ports[0])));
const context = new AudioContext({ sampleRate: ${SAMPLE_RATE} });
const processorLoaded = context.audioWorklet.addModule(URL.createObjectURL(
new Blob([ ${JSON.stringify(CAPTURE_PROCESSOR)} ], { type: "text/javascript" })));
async function pipe(source, media) {
- await processorLoaded;
+ const [ port ] = await Promise.all([ portReceived, processorLoaded ]);
const collector = new AudioWorkletNode(context, "chunk-collector", {
numberOfInputs: 1,
numberOfOutputs: 0,
@@ -176,7 +158,9 @@ async function pipe(source, media) {
channelCountMode: "explicit",
processorOptions: { chunkSamples: ${CHUNK_SAMPLES} }
});
- collector.port.onmessage = (e) => ipcRenderer.send("${SAMPLES_CHANNEL}", source, e.data);
+ // Copied, not transferred: a MessagePortMain receives a message with a transferred buffer
+ // as null.
+ collector.port.onmessage = (e) => port.postMessage({ source, samples: e.data });
context.createMediaStreamSource(media).connect(collector);
}
diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.ts
index 3112a9d6b0..fe8a5fc6ef 100644
--- a/apps/desktop/src/services/transcription/engine.ts
+++ b/apps/desktop/src/services/transcription/engine.ts
@@ -1,13 +1,10 @@
-import dataDirs from "@triliumnext/server/src/services/data_dir.js";
import fs from "fs";
import path from "path";
import type { SpeakerEmbeddingExtractor, Vad } from "sherpa-onnx-node";
+import { SAMPLE_RATE } from "./protocol";
import { SpeakerClusterer, type SpeakerClustererOptions } from "./speakers";
-/** Every source and engine works on mono 16 kHz samples. */
-export const SAMPLE_RATE = 16000;
-
/** Turns one utterance into text. */
export interface SpeechRecognizer {
transcribe(samples: Float32Array): Promise<string>;
@@ -47,11 +44,8 @@ export interface SpeechEngine {
settings: Record<string, unknown>;
}
-/** Where the models live: Parakeet TDT 0.6B v3 (int8) plus `silero_vad.onnx`. */
-export const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");
-
/** The speaker embedding model, optional: without it, utterances carry no speaker. */
-const SPEAKER_MODEL = path.join(MODEL_DIR, "speaker", "nemo_en_titanet_small.onnx");
+const SPEAKER_MODEL = path.join("speaker", "nemo_en_titanet_small.onnx");
/** Silero VAD consumes 512-sample windows at 16 kHz. */
const VAD_WINDOW_SIZE = 512;
@@ -85,33 +79,31 @@ const VAD_SETTINGS = {
};
/**
- * Loads the local engine: Silero VAD for segmentation and Parakeet for recognition, both through
- * sherpa-onnx. Recognition runs on the libuv thread pool, so it never blocks the main process.
+ * Loads the local engine from `modelDir`: Silero VAD for segmentation (`silero_vad.onnx`) and
+ * Parakeet TDT 0.6B v3 int8 for recognition, both through sherpa-onnx. Recognition runs on the
+ * libuv thread pool; the VAD and the speaker embeddings run on the calling thread.
*/
-export async function loadSherpaEngine(): Promise<SpeechEngine> {
- if (!fs.existsSync(MODEL_DIR)) {
- throw new ModelMissingError(MODEL_DIR);
- }
-
+export async function loadSherpaEngine(modelDir: string): Promise<SpeechEngine> {
+ const speakerModel = path.join(modelDir, SPEAKER_MODEL);
const mod = await import("sherpa-onnx-node");
const sherpa = mod.default ?? mod;
const recognizer = await sherpa.OfflineRecognizer.createAsync({
featConfig: { sampleRate: SAMPLE_RATE, featureDim: 80 },
modelConfig: {
transducer: {
- encoder: path.join(MODEL_DIR, "encoder.int8.onnx"),
- decoder: path.join(MODEL_DIR, "decoder.int8.onnx"),
- joiner: path.join(MODEL_DIR, "joiner.int8.onnx")
+ encoder: path.join(modelDir, "encoder.int8.onnx"),
+ decoder: path.join(modelDir, "decoder.int8.onnx"),
+ joiner: path.join(modelDir, "joiner.int8.onnx")
},
- tokens: path.join(MODEL_DIR, "tokens.txt"),
+ tokens: path.join(modelDir, "tokens.txt"),
...RECOGNIZER_SETTINGS
},
...DECODING_SETTINGS
});
- const speakerExtractor = fs.existsSync(SPEAKER_MODEL)
+ const speakerExtractor = fs.existsSync(speakerModel)
? new sherpa.SpeakerEmbeddingExtractor({
- model: SPEAKER_MODEL, numThreads: 1, provider: "cpu"
+ model: speakerModel, numThreads: 1, provider: "cpu"
})
: null;
@@ -125,7 +117,7 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {
},
createSegmenter: () => new VadSegmenter(new sherpa.Vad({
sileroVad: {
- model: path.join(MODEL_DIR, "silero_vad.onnx"),
+ model: path.join(modelDir, "silero_vad.onnx"),
...VAD_SETTINGS
},
sampleRate: SAMPLE_RATE,
@@ -136,22 +128,16 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {
? () => new EmbeddingSpeakerTracker(speakerExtractor, SPEAKER_SETTINGS)
: undefined,
settings: {
- modelDirectory: path.resolve(MODEL_DIR),
+ modelDirectory: path.resolve(modelDir),
recognizer: { ...RECOGNIZER_SETTINGS, ...DECODING_SETTINGS },
vad: VAD_SETTINGS,
speakers: speakerExtractor
- ? { model: path.basename(SPEAKER_MODEL), ...SPEAKER_SETTINGS }
+ ? { model: path.basename(speakerModel), ...SPEAKER_SETTINGS }
: null
}
};
}
-export class ModelMissingError extends Error {
- constructor(readonly directory: string) {
- super(`No speech model in ${directory}`);
- }
-}
-
class VadSegmenter implements SpeechSegmenter {
constructor(private readonly vad: Vad) {}
diff --git a/apps/desktop/src/services/transcription/index.spec.ts b/apps/desktop/src/services/transcription/index.spec.ts
index 9cd5bfdc2a..adb5e5ed3f 100644
--- a/apps/desktop/src/services/transcription/index.spec.ts
+++ b/apps/desktop/src/services/transcription/index.spec.ts
@@ -1,17 +1,19 @@
import type { TranscriptionState } from "@triliumnext/commons";
import { beforeEach, describe, expect, it, vi } from "vitest";
-import type { RecordingSessionOptions } from "./session";
+import type { WorkerListeners } from "./worker_client";
const h = vi.hoisted(() => ({
handlers: new Map<string, (...args: unknown[]) => unknown>(),
broadcast: [] as TranscriptionState[],
notes: new Set<string>([ "meeting" ]),
isDev: false,
- loadEngine: vi.fn(),
- sessionOptions: null as RecordingSessionOptions | null,
- sessionStart: vi.fn(),
- sessionStop: vi.fn()
+ isModelInstalled: true,
+ log: { info: vi.fn(), error: vi.fn() },
+ listeners: null as WorkerListeners | null,
+ worker: { start: vi.fn(), connectCapture: vi.fn(), stop: vi.fn(), kill: vi.fn() },
+ capture: { start: vi.fn(), stop: vi.fn() },
+ sink: { open: vi.fn(), write: vi.fn() }
}));
vi.mock("electron", () => ({
@@ -29,11 +31,16 @@ vi.mock("electron", () => ({
}
}
}))
+ },
+ MessageChannelMain: class {
+ port1 = "capture end";
+ port2 = "worker end";
}
}));
+vi.mock("fs", () => ({ default: { existsSync: () => h.isModelInstalled } }));
vi.mock("@triliumnext/core", () => ({
becca: { getNote: (noteId: string) => h.notes.has(noteId) ? { noteId } : null },
- getLog: () => ({ info: vi.fn(), error: vi.fn() }),
+ getLog: () => h.log,
utils: { isDev: () => h.isDev }
}));
vi.mock("i18next", () => ({
@@ -42,37 +49,35 @@ vi.mock("i18next", () => ({
vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({
default: { TRILIUM_DATA_DIR: "/data" }
}));
-vi.mock("./capture", () => ({ ElectronAudioCapture: class {} }));
-vi.mock("./engine", async (importOriginal) => ({
- ...(await importOriginal<typeof import("./engine")>()),
- loadSherpaEngine: h.loadEngine
-}));
-vi.mock("./attachment_sink", () => ({ AttachmentSink: class {} }));
-vi.mock("./audio_recorder", () => ({
- getRecordingDirectory: (createdAt: number, noteId: string) => `/recordings/${noteId}`,
- TranscriptFileSink: class {
- constructor(readonly directory: string) {}
- },
- WavRecorder: class {
- constructor(
- readonly directory: string, readonly noteId: string, readonly settings: object
- ) {}
+vi.mock("./capture", () => ({
+ CaptureWindow: class {
+ start = h.capture.start;
+ stop = h.capture.stop;
}
}));
-vi.mock("./session", () => ({
- RecordingSession: class {
- start = h.sessionStart;
- stop = h.sessionStop;
+vi.mock("./attachment_sink", () => ({
+ AttachmentSink: class {
+ open = h.sink.open;
+ write = h.sink.write;
- constructor(options: RecordingSessionOptions) {
- h.sessionOptions = options;
+ constructor(readonly noteId: string) {}
+ }
+}));
+vi.mock("./worker_client", () => ({
+ TranscriptionWorker: class {
+ start = h.worker.start;
+ connectCapture = h.worker.connectCapture;
+ stop = h.worker.stop;
+ kill = h.worker.kill;
+
+ constructor(listeners: WorkerListeners) {
+ h.listeners = listeners;
}
}
}));
-const { ModelMissingError } = await import("./engine");
-
const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };
+const STARTED_AT = Date.parse("2026-10-14T09:30:00.000Z");
async function loadService() {
vi.resetModules();
@@ -86,8 +91,10 @@ beforeEach(() => {
h.handlers.clear();
h.broadcast.length = 0;
h.isDev = false;
- h.loadEngine.mockResolvedValue({});
- h.sessionStart.mockResolvedValue([ "microphone", "system" ]);
+ h.isModelInstalled = true;
+ h.listeners = null;
+ h.worker.start.mockResolvedValue({ type: "ready", startedAt: STARTED_AT, settings: {} });
+ h.capture.start.mockResolvedValue([ "microphone", "system" ]);
});
describe("transcription IPC", () => {
@@ -101,7 +108,8 @@ describe("transcription IPC", () => {
.toEqual({ status: "recording", noteId: "meeting", pendingSources: [] });
await invoke("transcription-stop");
- expect(h.sessionStop).toHaveBeenCalledOnce();
+ expect(h.capture.stop.mock.invocationCallOrder[0])
+ .toBeLessThan(h.worker.stop.mock.invocationCallOrder[0]);
expect(h.broadcast.map((state) => state.status)).toEqual([
"starting", "starting", "recording", "recording",
"stopping", "stopping", "idle", "idle"
@@ -109,11 +117,37 @@ describe("transcription IPC", () => {
expect(await invoke("transcription-get-state")).toEqual(IDLE);
});
- it("tells every window which sources are pending while it records", async () => {
+ it("connects the capture window to the transcription process once that is ready", async () => {
+ const invoke = await loadService();
+ await invoke("transcription-start", "meeting");
+
+ expect(h.worker.start).toHaveBeenCalledExactlyOnceWith({
+ noteId: "meeting",
+ modelDirectory: "/data/transcription-model",
+ recordingDirectory: null
+ });
+ expect(h.sink.open).toHaveBeenCalledExactlyOnceWith(STARTED_AT);
+ expect(h.worker.connectCapture).toHaveBeenCalledExactlyOnceWith("worker end");
+ expect(h.capture.start).toHaveBeenCalledExactlyOnceWith("capture end");
+ expect(h.sink.open.mock.invocationCallOrder[0])
+ .toBeLessThan(h.capture.start.mock.invocationCallOrder[0]);
+ });
+
+ it("stores the utterances and tells every window which sources are pending", async () => {
const invoke = await loadService();
await invoke("transcription-start", "meeting");
+ const utterance = { source: "system" as const, offsetMs: 0, text: "Hi" };
+
+ h.listeners?.onUtterance(utterance);
+ h.sink.write.mockImplementationOnce(() => {
+ throw new Error("note deleted");
+ });
+ h.listeners?.onUtterance(utterance);
+ expect(h.sink.write).toHaveBeenCalledTimes(2);
+ expect(h.log.error).toHaveBeenCalledWith(
+ "Transcription: saving an utterance failed: Error: note deleted");
- h.sessionOptions?.onPendingChanged?.([ "system" ]);
+ h.listeners?.onPendingChanged([ "system" ]);
expect(h.broadcast.at(-1)).toEqual({
status: "recording", noteId: "meeting", pendingSources: [ "system" ]
});
@@ -126,46 +160,66 @@ describe("transcription IPC", () => {
});
it("keeps the audio and the transcript as produced only in development builds", async () => {
- const settings = { recognizer: { decodingMethod: "greedy_search" } };
- h.loadEngine.mockResolvedValue({ settings });
+ vi.useFakeTimers({ now: STARTED_AT, toFake: [ "Date" ] });
const invoke = await loadService();
await invoke("transcription-start", "meeting");
- expect(h.sessionOptions?.sinks).toHaveLength(1);
- expect(h.sessionOptions?.taps).toEqual([]);
+ expect(h.worker.start.mock.calls[0][0].recordingDirectory).toBeNull();
await invoke("transcription-stop");
h.isDev = true;
await invoke("transcription-start", "meeting");
- expect(h.sessionOptions?.sinks[1]).toEqual({ directory: "/recordings/meeting" });
- expect(h.sessionOptions?.taps).toEqual([
- { directory: "/recordings/meeting", noteId: "meeting", settings }
- ]);
+ expect(h.worker.start.mock.calls[1][0].recordingDirectory)
+ .toBe("/data/transcription-recordings/2026-10-14T09-30-00-000Z_meeting");
+ vi.useRealTimers();
});
- it("refuses a note that does not exist, without changing state", async () => {
+ it("refuses a missing note or model, without changing state", async () => {
const invoke = await loadService();
expect(await invoke("transcription-start", "deleted")).toBe("transcription.note-not-found");
expect(await invoke("transcription-start", { noteId: "meeting" }))
.toBe("transcription.note-not-found");
- expect(h.loadEngine).not.toHaveBeenCalled();
+ h.isModelInstalled = false;
+ expect(await invoke("transcription-start", "meeting"))
+ .toBe(`transcription.model-missing {"directory":"/data/transcription-model"}`);
+ expect(h.listeners).toBeNull();
expect(h.broadcast).toEqual([]);
});
- it("explains why it could not start and returns to idle", async () => {
+ it("explains why it could not start, ends the process and returns to idle", async () => {
const invoke = await loadService();
- h.loadEngine.mockRejectedValueOnce(new ModelMissingError("/data/transcription-model"));
+ h.worker.start.mockRejectedValueOnce(new Error("Protobuf parsing failed"));
expect(await invoke("transcription-start", "meeting"))
- .toBe(`transcription.model-missing {"directory":"/data/transcription-model"}`);
+ .toBe(`transcription.start-failed {"message":"Protobuf parsing failed"}`);
+ expect(h.capture.start).not.toHaveBeenCalled();
- h.sessionStart.mockRejectedValueOnce(new Error("No audio source could be captured."));
+ h.capture.start.mockRejectedValueOnce(new Error("No audio source could be captured."));
expect(await invoke("transcription-start", "meeting"))
.toBe(`transcription.start-failed {"message":"No audio source could be captured."}`);
+ expect(h.worker.kill).toHaveBeenCalledTimes(2);
+ expect(h.capture.stop).toHaveBeenCalledTimes(2);
expect(await invoke("transcription-get-state")).toEqual(IDLE);
await invoke("transcription-stop");
- expect(h.sessionStop).not.toHaveBeenCalled();
+ expect(h.worker.stop).not.toHaveBeenCalled();
+ });
+
+ it("ends a recording whose transcription process died, and says why once", async () => {
+ const invoke = await loadService();
+ await invoke("transcription-start", "meeting");
+
+ h.listeners?.onUnexpectedExit(139);
+ expect(h.capture.stop).toHaveBeenCalledOnce();
+ expect(h.broadcast.at(-1))
+ .toEqual({ ...IDLE, error: "transcription.stopped-unexpectedly" });
+ expect(await invoke("transcription-get-state")).toEqual(IDLE);
+ expect(h.log.error).toHaveBeenCalledWith(
+ "Transcription: the transcription process exited with code 139.");
+
+ await invoke("transcription-stop");
+ expect(h.worker.stop).not.toHaveBeenCalled();
+ expect(await invoke("transcription-start", "meeting")).toBe("");
});
});
diff --git a/apps/desktop/src/services/transcription/index.ts b/apps/desktop/src/services/transcription/index.ts
index c6c77987c6..85dac744be 100644
--- a/apps/desktop/src/services/transcription/index.ts
+++ b/apps/desktop/src/services/transcription/index.ts
@@ -1,18 +1,30 @@
-import type { TranscriptionState } from "@triliumnext/commons";
+import type { TranscriptionState, TranscriptUtterance } from "@triliumnext/commons";
import { becca, getLog, utils as coreUtils } from "@triliumnext/core";
-import { BrowserWindow, ipcMain } from "electron";
+import dataDirs from "@triliumnext/server/src/services/data_dir.js";
+import { BrowserWindow, ipcMain, MessageChannelMain } from "electron";
+import fs from "fs";
import { t } from "i18next";
+import path from "path";
import { AttachmentSink } from "./attachment_sink";
-import { getRecordingDirectory, TranscriptFileSink, WavRecorder } from "./audio_recorder";
-import { ElectronAudioCapture } from "./capture";
-import { loadSherpaEngine, ModelMissingError, type SpeechEngine } from "./engine";
-import { type AudioTap, RecordingSession, type TranscriptSink } from "./session";
+import { CaptureWindow } from "./capture";
+import { TranscriptionWorker } from "./worker_client";
+
+/** Where the models live: Parakeet TDT 0.6B v3 (int8) plus `silero_vad.onnx`. */
+export const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-model");
+
+/** Where development builds keep the audio of each recording, one directory per recording. */
+export const RECORDINGS_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-recordings");
const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };
+interface Recording {
+ worker: TranscriptionWorker;
+ capture: CaptureWindow;
+}
+
let state = IDLE;
-let activeSession: RecordingSession | null = null;
+let activeRecording: Recording | null = null;
export function setupTranscriptionHandlers() {
ipcMain.handle("transcription-start", (_event, noteId: unknown) => startRecording(noteId));
@@ -21,7 +33,8 @@ export function setupTranscriptionHandlers() {
}
/**
- * Starts recording for the meeting note `noteId`.
+ * Starts recording for the meeting note `noteId`. The engine runs in a utility process, which
+ * the capture window posts its samples to; the main process only stores the utterances.
*
* @returns An empty string once recording runs, or the reason it cannot start.
*/
@@ -32,41 +45,56 @@ export async function startRecording(noteId: unknown): Promise<string> {
if (typeof noteId !== "string" || !becca.getNote(noteId)) {
return t("transcription.note-not-found");
}
+ if (!fs.existsSync(MODEL_DIR)) {
+ return t("transcription.model-missing", { directory: MODEL_DIR });
+ }
setState({ status: "starting", noteId, pendingSources: [] });
+ const sink = new AttachmentSink(noteId);
+ const recording: Recording = {
+ capture: new CaptureWindow(),
+ worker: new TranscriptionWorker({
+ onUtterance: (utterance) => writeUtterance(sink, utterance),
+ onPendingChanged: (pendingSources) => setState({ ...state, pendingSources }),
+ onUnexpectedExit: (code) => handleUnexpectedExit(recording, code),
+ log: getLog()
+ })
+ };
+ activeRecording = recording;
try {
- const engine = await loadSherpaEngine();
- const session = new RecordingSession({
- capture: new ElectronAudioCapture(),
- engine,
- ...createReceivers(noteId, engine),
- onPendingChanged: (pendingSources) => setState({ ...state, pendingSources })
+ const { startedAt } = await recording.worker.start({
+ noteId,
+ modelDirectory: MODEL_DIR,
+ recordingDirectory: coreUtils.isDev() ? getRecordingDirectory(Date.now(), noteId) : null
});
- const sources = await session.start();
- activeSession = session;
+ sink.open(startedAt);
+ const { port1, port2 } = new MessageChannelMain();
+ recording.worker.connectCapture(port2);
+ const sources = await recording.capture.start(port1);
getLog().info(`Transcription: recording ${sources.join(" and ")} for note ${noteId}.`);
setState({ ...state, status: "recording" });
return "";
} catch (e) {
+ activeRecording = null;
+ recording.capture.stop();
+ recording.worker.kill();
setState(IDLE);
- if (e instanceof ModelMissingError) {
- return t("transcription.model-missing", { directory: e.directory });
- }
getLog().error(`Transcription did not start: ${(e as Error)?.stack ?? e}`);
return t("transcription.start-failed", { message: (e as Error)?.message ?? String(e) });
}
}
export async function stopRecording() {
- const session = activeSession;
- if (state.status !== "recording" || !session) {
+ const recording = activeRecording;
+ if (state.status !== "recording" || !recording) {
return;
}
setState({ ...state, status: "stopping" });
- activeSession = null;
+ activeRecording = null;
+ recording.capture.stop();
try {
- await session.stop();
+ await recording.worker.stop();
getLog().info(`Transcription: recording for note ${state.noteId} stopped.`);
} finally {
setState(IDLE);
@@ -77,24 +105,42 @@ export function getState() {
return state;
}
+/** The directory under {@link RECORDINGS_DIR} for a recording of `noteId` begun at `createdAt`. */
+export function getRecordingDirectory(createdAt: number, noteId: string) {
+ const timestamp = new Date(createdAt).toISOString().replace(/[:.]/g, "-");
+ return path.join(RECORDINGS_DIR, `${timestamp}_${noteId}`);
+}
+
+function writeUtterance(sink: AttachmentSink, utterance: TranscriptUtterance) {
+ try {
+ sink.write(utterance);
+ } catch (e) {
+ getLog().error(`Transcription: saving an utterance failed: ${e}`);
+ }
+}
+
/**
- * Where the utterances and the audio of a recording for `noteId` go. Development builds also keep
- * the audio and the transcript as produced, to transcribe the audio again while the engine is tuned.
+ * Ends a recording whose transcription process died, keeping what was transcribed, and tells
+ * every window why it ended.
*/
-function createReceivers(noteId: string, engine: SpeechEngine) {
- const sinks: TranscriptSink[] = [ new AttachmentSink(noteId) ];
- const taps: AudioTap[] = [];
- if (coreUtils.isDev()) {
- const directory = getRecordingDirectory(Date.now(), noteId);
- sinks.push(new TranscriptFileSink(directory));
- taps.push(new WavRecorder(directory, noteId, engine.settings));
+function handleUnexpectedExit(recording: Recording, code: number) {
+ if (activeRecording !== recording) {
+ return;
}
- return { sinks, taps };
+ activeRecording = null;
+ recording.capture.stop();
+ getLog().error(`Transcription: the transcription process exited with code ${code}.`);
+ state = IDLE;
+ broadcast({ ...IDLE, error: t("transcription.stopped-unexpectedly") });
}
function setState(newState: TranscriptionState) {
state = newState;
+ broadcast(state);
+}
+
+function broadcast(stateToSend: TranscriptionState) {
for (const window of BrowserWindow.getAllWindows()) {
- window.webContents.send("transcription-state", state);
+ window.webContents.send("transcription-state", stateToSend);
}
}
diff --git a/apps/desktop/src/services/transcription/protocol.ts b/apps/desktop/src/services/transcription/protocol.ts
new file mode 100644
index 0000000000..dfbb8e1d42
--- /dev/null
+++ b/apps/desktop/src/services/transcription/protocol.ts
@@ -0,0 +1,40 @@
+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
+
+/** Every source and engine works on mono 16 kHz samples. */
+export const SAMPLE_RATE = 16000;
+
+/** Where the transcription process reports to, since it has no access to Trilium's log. */
+export interface Logger {
+ info(message: string): void;
+ error(message: string): void;
+}
+
+/** What the main process sends the transcription process, in this order. */
+export type WorkerRequest =
+ | {
+ type: "start";
+ noteId: string;
+ modelDirectory: string;
+ /** Where development builds keep the audio and the transcript as produced, else `null`. */
+ recordingDirectory: string | null;
+ }
+ /** Sent once the recording runs, with the port the capture page posts its samples to. */
+ | { type: "capture" }
+ | { type: "stop" };
+
+/** What the transcription process reports to the main process. */
+export type WorkerEvent =
+ /** The engine is loaded and the recording's start time is set; capturing can begin. */
+ | { type: "ready"; startedAt: number; settings: Record<string, unknown> }
+ | { type: "failed"; message: string }
+ | { type: "pending"; sources: TranscriptSource[] }
+ | { type: "utterance"; utterance: TranscriptUtterance }
+ | { type: "log"; level: keyof Logger; message: string }
+ /** Every utterance captured before `stop` has been sent. */
+ | { type: "stopped" };
+
+/** One chunk of samples, posted by the capture page straight to the transcription process. */
+export interface CaptureChunk {
+ source: TranscriptSource;
+ samples: Float32Array;
+}
diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.ts
index 7b5545f74c..0dc605d1d8 100644
--- a/apps/desktop/src/services/transcription/session.spec.ts
+++ b/apps/desktop/src/services/transcription/session.spec.ts
@@ -1,15 +1,12 @@
import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
import { beforeEach, describe, expect, it, vi } from "vitest";
-import type { SamplesListener } from "./capture";
import type { SpeakerTracker, SpeechSegment } from "./engine";
-import { type AudioTap, RecordingSession, type TranscriptSink } from "./session";
+import {
+ type AudioTap, RecordingSession, type SamplesListener, type TranscriptSink
+} from "./session";
-const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));
-vi.mock("@triliumnext/core", () => ({ getLog: () => log }));
-vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({
- default: { TRILIUM_DATA_DIR: "/data" }
-}));
+const log = { info: vi.fn(), error: vi.fn() };
/**
* A chunk the fake segmenter reads as a finished utterance: `id` becomes the transcribed text
@@ -36,9 +33,8 @@ function createHarness(extraSinks: TranscriptSink[] = [], taps: AudioTap[] = [])
let clock = 1000;
let onSamples: SamplesListener = () => {};
const capture = {
- start: vi.fn(async (listener: SamplesListener) => {
+ start: vi.fn((listener: SamplesListener) => {
onSamples = listener;
- return [ "microphone", "system" ] as TranscriptSource[];
}),
stop: vi.fn()
};
@@ -77,7 +73,7 @@ function createHarness(extraSinks: TranscriptSink[] = [], taps: AudioTap[] = [])
const sinks = [ ...extraSinks, sink ];
const onPendingChanged = vi.fn();
const session = new RecordingSession({
- capture, engine, sinks, taps, onPendingChanged, now: () => clock
+ capture, engine, sinks, taps, log, onPendingChanged, now: () => clock
});
return {
@@ -99,6 +95,7 @@ describe("RecordingSession", () => {
const { session, capture, engine, sink, deliver, written } = createHarness();
await session.start();
expect(sink.open).toHaveBeenCalledExactlyOnceWith(1000);
+ expect(session.startedAt).toBe(1000);
expect(sink.open.mock.invocationCallOrder[0])
.toBeLessThan(capture.start.mock.invocationCallOrder[0]);
@@ -237,7 +234,7 @@ describe("RecordingSession", () => {
"Transcription: identifying the speaker on system failed: Error: model crashed");
});
- it("logs a failing source instead of throwing into the IPC listener", async () => {
+ it("logs a failing source instead of throwing into the capture", async () => {
const { session, engine, deliver } = createHarness();
engine.createSegmenter.mockImplementationOnce(() => {
throw new Error("VAD model missing");
diff --git a/apps/desktop/src/services/transcription/session.ts b/apps/desktop/src/services/transcription/session.ts
index e733b6c9b2..2019e0a87d 100644
--- a/apps/desktop/src/services/transcription/session.ts
+++ b/apps/desktop/src/services/transcription/session.ts
@@ -1,14 +1,20 @@
import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
-import { getLog } from "@triliumnext/core";
-import type { AudioCapture } from "./capture";
-import {
- SAMPLE_RATE, type SpeakerTracker, type SpeechEngine, type SpeechSegment, type SpeechSegmenter
-} from "./engine";
+import type { SpeakerTracker, SpeechEngine, SpeechSegment, SpeechSegmenter } from "./engine";
+import { type Logger, SAMPLE_RATE } from "./protocol";
/** The sources whose speakers are told apart; the microphone carries only the local speaker. */
const DIARIZED_SOURCES: readonly TranscriptSource[] = [ "system" ];
+export type SamplesListener = (source: TranscriptSource, samples: Float32Array) => void;
+
+/** Produces mono {@link SAMPLE_RATE} samples per source until stopped. */
+export interface AudioCapture {
+ /** Starts delivering samples to `onSamples`. */
+ start(onSamples: SamplesListener): void;
+ stop(): void;
+}
+
/** Receives the utterances of a recording in the order they were spoken per source. */
export interface TranscriptSink {
/**
@@ -39,6 +45,7 @@ export interface RecordingSessionOptions {
engine: SpeechEngine;
sinks: TranscriptSink[];
taps?: AudioTap[];
+ log: Logger;
/**
* Called with the sources whose speech has not reached the sinks yet, either because it is
* still being spoken or because it waits to be transcribed, every time that list changes.
@@ -58,25 +65,30 @@ export class RecordingSession {
/** How many utterances of each source wait to be transcribed or written. */
private readonly queued = new Map<TranscriptSource, number>();
private queue = Promise.resolve();
- private startedAt = 0;
+ private startedAtMs = 0;
private pendingSources: TranscriptSource[] = [];
constructor(private readonly options: RecordingSessionOptions) {}
+ /** When the recording started, in milliseconds since the epoch; set by {@link start}. */
+ get startedAt() {
+ return this.startedAtMs;
+ }
+
async start() {
- this.startedAt = this.now();
+ this.startedAtMs = this.now();
for (const receiver of [ ...this.options.sinks, ...this.options.taps ?? [] ]) {
- await receiver.open?.(this.startedAt);
+ await receiver.open?.(this.startedAtMs);
}
- return await this.options.capture.start((source, samples) => {
- // An exception thrown from an `ipcMain` listener takes down the main process.
+ this.options.capture.start((source, samples) => {
+ // An exception thrown from the capture's listener ends the transcription process.
try {
const pipeline = this.pipelineFor(source, samples);
this.tap(source, samples, pipeline.firstSampleOffsetMs);
pipeline.accept(samples);
this.updatePending();
} catch (e) {
- getLog().error(`Transcription: ${source} failed: ${(e as Error)?.message ?? e}`);
+ this.log.error(`Transcription: ${source} failed: ${(e as Error)?.message ?? e}`);
}
});
}
@@ -92,7 +104,7 @@ export class RecordingSession {
try {
await tap.close?.();
} catch (e) {
- getLog().error(`Transcription: closing an audio tap failed: ${e}`);
+ this.log.error(`Transcription: closing an audio tap failed: ${e}`);
}
}
await this.queue;
@@ -101,7 +113,7 @@ export class RecordingSession {
try {
await sink.close?.();
} catch (e) {
- getLog().error(`Transcription: closing a sink failed: ${e}`);
+ this.log.error(`Transcription: closing a sink failed: ${e}`);
}
}
}
@@ -111,7 +123,7 @@ export class RecordingSession {
try {
tap.write(source, samples, firstSampleOffsetMs);
} catch (e) {
- getLog().error(`Transcription: an audio tap failed on ${source}: ${e}`);
+ this.log.error(`Transcription: an audio tap failed on ${source}: ${e}`);
}
}
}
@@ -124,8 +136,9 @@ export class RecordingSession {
pipeline = new SourcePipeline(
source,
this.options.engine.createSegmenter(),
- Math.max(0, this.now() - this.startedAt - chunkMs),
- (offsetMs, samples) => this.enqueue(source, offsetMs, samples)
+ Math.max(0, this.now() - this.startedAtMs - chunkMs),
+ (offsetMs, samples) => this.enqueue(source, offsetMs, samples),
+ this.options.log
);
this.pipelines.set(source, pipeline);
const tracker = DIARIZED_SOURCES.includes(source)
@@ -155,21 +168,21 @@ export class RecordingSession {
try {
await sink.write(utterance);
} catch (e) {
- getLog().error(`Transcription: a sink failed: ${e}`);
+ this.log.error(`Transcription: a sink failed: ${e}`);
}
}
- }).catch((e) => getLog().error(`Transcription failed: ${e}`)).finally(() => {
+ }).catch((e) => this.log.error(`Transcription failed: ${e}`)).finally(() => {
this.queued.set(source, (this.queued.get(source) ?? 1) - 1);
this.updatePending();
});
}
- /** Runs on the main thread, synchronously: about 100 ms per utterance. */
+ /** Runs synchronously, about 100 ms per utterance, so the next chunks wait meanwhile. */
private identifySpeaker(source: TranscriptSource, samples: Float32Array) {
try {
return this.speakerTrackers.get(source)?.identify(samples) ?? null;
} catch (e) {
- getLog().error(`Transcription: identifying the speaker on ${source} failed: ${e}`);
+ this.log.error(`Transcription: identifying the speaker on ${source} failed: ${e}`);
return null;
}
}
@@ -189,6 +202,10 @@ export class RecordingSession {
private now() {
return (this.options.now ?? Date.now)();
}
+
+ private get log() {
+ return this.options.log;
+ }
}
/** Segments one source and logs its peak level every 5 s, to spot a silent source. */
@@ -201,7 +218,8 @@ class SourcePipeline {
private readonly segmenter: SpeechSegmenter,
/** Milliseconds from the start of the recording to the source's first sample. */
readonly firstSampleOffsetMs: number,
- private readonly onSegment: (offsetMs: number, samples: Float32Array) => void
+ private readonly onSegment: (offsetMs: number, samples: Float32Array) => void,
+ private readonly log: Logger
) {}
accept(samples: Float32Array) {
@@ -230,7 +248,7 @@ class SourcePipeline {
}
this.peakSampleCount += samples.length;
if (this.peakSampleCount >= 5 * SAMPLE_RATE) {
- getLog().info(`Transcription: ${this.source} peak ${this.peak.toFixed(4)} over 5 s`);
+ this.log.info(`Transcription: ${this.source} peak ${this.peak.toFixed(4)} over 5 s`);
this.peak = 0;
this.peakSampleCount = 0;
}
diff --git a/apps/desktop/src/services/transcription/transcription_worker.ts b/apps/desktop/src/services/transcription/transcription_worker.ts
new file mode 100644
index 0000000000..abe6bfbea5
--- /dev/null
+++ b/apps/desktop/src/services/transcription/transcription_worker.ts
@@ -0,0 +1,113 @@
+/**
+ * The transcription process: an Electron utility process that runs one recording's VAD, speaker
+ * embeddings and recognizer, so none of them takes time from the main process. The capture page
+ * posts its samples here directly; the main process only receives the utterances. Started by
+ * `TranscriptionWorker` in `worker_client.ts`, which also ends it.
+ */
+import type { TranscriptSource } from "@triliumnext/commons";
+import type { MessagePortMain } from "electron";
+
+import { TranscriptFileSink, WavRecorder } from "./audio_recorder";
+import { loadSherpaEngine } from "./engine";
+import type { CaptureChunk, Logger, WorkerEvent, WorkerRequest } from "./protocol";
+import {
+ type AudioCapture, type AudioTap, RecordingSession, type SamplesListener, type TranscriptSink
+} from "./session";
+
+const log: Logger = {
+ info: (message) => post({ type: "log", level: "info", message }),
+ error: (message) => post({ type: "log", level: "error", message })
+};
+
+type StartRequest = Extract<WorkerRequest, { type: "start" }>;
+
+let session: RecordingSession | null = null;
+let capture: PortCapture | null = null;
+
+process.parentPort.on("message", ({ data, ports }) => {
+ const request = data as WorkerRequest;
+ switch (request.type) {
+ case "start":
+ start(request).catch((e) => {
+ post({ type: "failed", message: (e as Error)?.message ?? String(e) });
+ });
+ break;
+ case "capture":
+ if (ports[0]) {
+ capture?.connect(ports[0]);
+ }
+ break;
+ case "stop":
+ void stop();
+ break;
+ }
+});
+
+async function start({ noteId, modelDirectory, recordingDirectory }: StartRequest) {
+ const engine = await loadSherpaEngine(modelDirectory);
+ const sinks: TranscriptSink[] = [
+ { write: (utterance) => post({ type: "utterance", utterance }) }
+ ];
+ const taps: AudioTap[] = [];
+ if (recordingDirectory) {
+ sinks.push(new TranscriptFileSink(recordingDirectory));
+ taps.push(new WavRecorder(recordingDirectory, noteId, engine.settings, log));
+ }
+
+ capture = new PortCapture();
+ const newSession = new RecordingSession({
+ capture,
+ engine,
+ sinks,
+ taps,
+ log,
+ onPendingChanged: (sources) => post({ type: "pending", sources })
+ });
+ await newSession.start();
+ session = newSession;
+ post({ type: "ready", startedAt: newSession.startedAt, settings: engine.settings });
+}
+
+async function stop() {
+ try {
+ await session?.stop();
+ } catch (e) {
+ log.error(`Transcription: stopping failed: ${e}`);
+ }
+ post({ type: "stopped" });
+}
+
+function post(event: WorkerEvent) {
+ process.parentPort.postMessage(event);
+}
+
+/** The samples the capture page posts to the port the main process connects. */
+class PortCapture implements AudioCapture {
+ private port: MessagePortMain | null = null;
+ private onSamples: SamplesListener | null = null;
+
+ start(onSamples: SamplesListener) {
+ this.onSamples = onSamples;
+ }
+
+ connect(port: MessagePortMain) {
+ this.port = port;
+ port.on("message", ({ data }) => {
+ const chunk = data as CaptureChunk | null;
+ if (chunk && isSource(chunk.source) && chunk.samples instanceof Float32Array) {
+ this.onSamples?.(chunk.source, chunk.samples);
+ }
+ });
+ port.start();
+ }
+
+ stop() {
+ this.onSamples = null;
+ this.port?.close();
+ this.port = null;
+ }
+}
+
+function isSource(source: unknown): source is TranscriptSource {
+ return source === "microphone" || source === "system";
+}
diff --git a/apps/desktop/src/services/transcription/worker_client.spec.ts b/apps/desktop/src/services/transcription/worker_client.spec.ts
new file mode 100644
index 0000000000..edabc231da
--- /dev/null
+++ b/apps/desktop/src/services/transcription/worker_client.spec.ts
@@ -0,0 +1,140 @@
+import { EventEmitter } from "events";
+import path from "path";
+import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from "vitest";
+
+import type { Logger, WorkerEvent } from "./protocol";
+import { TranscriptionWorker, type WorkerListeners } from "./worker_client";
+
+const h = vi.hoisted(() => ({
+ fork: vi.fn(),
+ hasBundle: false
+}));
+
+vi.mock("electron", () => ({ utilityProcess: { fork: h.fork } }));
+vi.mock("fs", () => ({
+ default: { existsSync: (file: string) => !file.endsWith(".cjs") || h.hasBundle }
+}));
+
+class FakeProcess extends EventEmitter {
+ postMessage = vi.fn();
+ kill = vi.fn(() => {
+ this.emit("exit", 0);
+ return true;
+ });
+
+ report(event: WorkerEvent) {
+ this.emit("message", event);
+ }
+}
+
+let child: FakeProcess;
+let listeners: {
+ [ name in keyof Omit<WorkerListeners, "log"> ]: Mock<WorkerListeners[name]>
+} & { log: { info: Mock<Logger["info"]>; error: Mock<Logger["error"]> } };
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ h.hasBundle = false;
+ child = new FakeProcess();
+ h.fork.mockReturnValue(child);
+ listeners = {
+ onUtterance: vi.fn(),
+ onPendingChanged: vi.fn(),
+ onUnexpectedExit: vi.fn(),
+ log: { info: vi.fn(), error: vi.fn() }
+ };
+});
+
+afterEach(() => {
+ vi.useRealTimers();
+});
+
+const REQUEST = { noteId: "meeting", modelDirectory: "/models", recordingDirectory: null };
+
+describe("TranscriptionWorker", () => {
+ it("starts the bundled worker, or the sources when there is no bundle", () => {
+ new TranscriptionWorker(listeners);
+ h.hasBundle = true;
+ new TranscriptionWorker(listeners);
+
+ expect(h.fork.mock.calls.map(([ file ]) => path.basename(file as string)))
+ .toEqual([ "transcription_worker.ts", "transcription_worker.cjs" ]);
+ });
+
+ it("resolves once the engine is ready, then relays what the process reports", async () => {
+ const worker = new TranscriptionWorker(listeners);
+ const started = worker.start(REQUEST);
+ expect(child.postMessage).toHaveBeenCalledExactlyOnceWith({ type: "start", ...REQUEST });
+
+ child.report({ type: "ready", startedAt: 1000, settings: { vad: {} } });
+ await expect(started).resolves.toEqual({
+ type: "ready", startedAt: 1000, settings: { vad: {} }
+ });
+
+ const port = { start: vi.fn() };
+ worker.connectCapture(port as never);
+ expect(child.postMessage).toHaveBeenLastCalledWith({ type: "capture" }, [ port ]);
+
+ const utterance = { source: "system" as const, offsetMs: 0, text: "Hi" };
+ child.report({ type: "utterance", utterance });
+ child.report({ type: "pending", sources: [ "system" ] });
+ child.report({ type: "log", level: "error", message: "system failed" });
+ expect(listeners.onUtterance).toHaveBeenCalledExactlyOnceWith(utterance);
+ expect(listeners.onPendingChanged).toHaveBeenCalledExactlyOnceWith([ "system" ]);
+ expect(listeners.log.error).toHaveBeenCalledExactlyOnceWith("system failed");
+ });
+
+ it("rejects the start when the engine fails or the process dies first", async () => {
+ const failing = new TranscriptionWorker(listeners).start(REQUEST);
+ child.report({ type: "failed", message: "Protobuf parsing failed" });
+ await expect(failing).rejects.toThrow("Protobuf parsing failed");
+
+ child = new FakeProcess();
+ h.fork.mockReturnValue(child);
+ const crashing = new TranscriptionWorker(listeners).start(REQUEST);
+ child.emit("exit", 139);
+ await expect(crashing).rejects.toThrow("The transcription process exited with code 139.");
+ expect(listeners.onUnexpectedExit).not.toHaveBeenCalled();
+ });
+
+ it("stops once the process has reported everything, then ends it", async () => {
+ const worker = new TranscriptionWorker(listeners);
+ let isStopped = false;
+ const stopped = worker.stop().then(() => {
+ isStopped = true;
+ });
+ expect(child.postMessage).toHaveBeenCalledExactlyOnceWith({ type: "stop" });
+ await Promise.resolve();
+ expect(isStopped).toBe(false);
+
+ child.report({ type: "stopped" });
+ await stopped;
+ expect(child.kill).toHaveBeenCalledOnce();
+ expect(listeners.onUnexpectedExit).not.toHaveBeenCalled();
+ await worker.stop();
+ expect(child.postMessage).toHaveBeenCalledOnce();
+ });
+
+ it("ends a process that does not stop in time", async () => {
+ vi.useFakeTimers();
+ const worker = new TranscriptionWorker(listeners);
+ const stopped = worker.stop();
+
+ await vi.advanceTimersByTimeAsync(2 * 60 * 1000);
+ await stopped;
+ expect(child.kill).toHaveBeenCalledOnce();
+ expect(listeners.log.error).toHaveBeenCalledExactlyOnceWith(
+ "Transcription: the transcription process did not stop in time.");
+ });
+
+ it("reports a process that exits while recording, but not one it ended", () => {
+ new TranscriptionWorker(listeners);
+ child.emit("exit", 139);
+ expect(listeners.onUnexpectedExit).toHaveBeenCalledExactlyOnceWith(139);
+
+ child = new FakeProcess();
+ h.fork.mockReturnValue(child);
+ new TranscriptionWorker(listeners).kill();
+ expect(listeners.onUnexpectedExit).toHaveBeenCalledOnce();
+ });
+});
diff --git a/apps/desktop/src/services/transcription/worker_client.ts b/apps/desktop/src/services/transcription/worker_client.ts
new file mode 100644
index 0000000000..284d63b12e
--- /dev/null
+++ b/apps/desktop/src/services/transcription/worker_client.ts
@@ -0,0 +1,129 @@
+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
+import { type MessagePortMain, utilityProcess, type UtilityProcess } from "electron";
+import fs from "fs";
+import path from "path";
+
+import type { Logger, WorkerEvent, WorkerRequest } from "./protocol";
+
+export interface WorkerListeners {
+ onUtterance(utterance: TranscriptUtterance): void;
+ onPendingChanged(sources: TranscriptSource[]): void;
+ /** Called when the process ends without having been stopped, as when it crashes. */
+ onUnexpectedExit(code: number): void;
+ log: Logger;
+}
+
+type ReadyEvent = Extract<WorkerEvent, { type: "ready" }>;
+type StartRequest = Extract<WorkerRequest, { type: "start" }>;
+
+/** How long {@link TranscriptionWorker.stop} waits for the speech still queued. */
+const STOP_TIMEOUT_MS = 2 * 60 * 1000;
+
+/**
+ * The transcription process (`transcription_worker.ts`), from the main process: starts it,
+ * connects the capture page to it and relays what it reports. One instance runs one recording;
+ * the process ends with it.
+ */
+export class TranscriptionWorker {
+ private readonly child: UtilityProcess;
+ private startRequest: { resolve(event: ReadyEvent): void; reject(error: Error): void } | null =
+ null;
+ private isStopping = false;
+ private hasExited = false;
+
+ constructor(private readonly listeners: WorkerListeners) {
+ this.child = utilityProcess.fork(getWorkerEntry(), [], {
+ serviceName: "Trilium transcription"
+ });
+ this.child.on("message", (event: WorkerEvent) => this.handle(event));
+ this.child.on("exit", (code) => this.handleExit(code));
+ }
+
+ /** Loads the engine and starts the recording, without capturing yet. */
+ start(request: Omit<StartRequest, "type">) {
+ return new Promise<ReadyEvent>((resolve, reject) => {
+ this.startRequest = { resolve, reject };
+ this.child.postMessage({ type: "start", ...request } satisfies WorkerRequest);
+ });
+ }
+
+ /** Hands over the port the capture page posts its samples to. */
+ connectCapture(port: MessagePortMain) {
+ this.child.postMessage({ type: "capture" } satisfies WorkerRequest, [ port ]);
+ }
+
+ /**
+ * Waits until the speech already captured has been reported, then ends the process. Ends it
+ * regardless after {@link STOP_TIMEOUT_MS}.
+ */
+ async stop() {
+ if (this.hasExited) {
+ return;
+ }
+ this.isStopping = true;
+ const exited = new Promise<void>((resolve) => this.child.once("exit", () => resolve()));
+ const timeout = setTimeout(() => {
+ this.listeners.log.error(
+ "Transcription: the transcription process did not stop in time.");
+ this.child.kill();
+ }, STOP_TIMEOUT_MS);
+ this.child.postMessage({ type: "stop" } satisfies WorkerRequest);
+ await exited;
+ clearTimeout(timeout);
+ }
+
+ /** Ends the process at once, dropping what it has not reported. */
+ kill() {
+ this.isStopping = true;
+ this.child.kill();
+ }
+
+ private handle(event: WorkerEvent) {
+ switch (event.type) {
+ case "ready":
+ this.startRequest?.resolve(event);
+ this.startRequest = null;
+ break;
+ case "failed":
+ this.startRequest?.reject(new Error(event.message));
+ this.startRequest = null;
+ break;
+ case "pending":
+ this.listeners.onPendingChanged(event.sources);
+ break;
+ case "utterance":
+ this.listeners.onUtterance(event.utterance);
+ break;
+ case "log":
+ this.listeners.log[event.level](event.message);
+ break;
+ case "stopped":
+ this.child.kill();
+ break;
+ }
+ }
+
+ private handleExit(code: number) {
+ this.hasExited = true;
+ const error = new Error(`The transcription process exited with code ${code}.`);
+ if (this.startRequest) {
+ this.startRequest.reject(error);
+ this.startRequest = null;
+ } else if (!this.isStopping) {
+ this.listeners.onUnexpectedExit(code);
+ }
+ }
+}
+
+/**
+ * The bundled `transcription_worker.cjs` beside the main bundle, or the TypeScript when running
+ * from sources, which the utility process reads through the `--import tsx` it inherits in
+ * `NODE_OPTIONS`.
+ */
+function getWorkerEntry() {
+ const candidates = [
+ path.join(__dirname, "transcription_worker.cjs"),
+ path.join(__dirname, "transcription_worker.ts")
+ ];
+ return candidates.find((candidate) => fs.existsSync(candidate)) ?? candidates[0];
+}
diff --git a/apps/server/src/assets/translations/en/server.json b/apps/server/src/assets/translations/en/server.json
index 20f20c845b..0bfb215580 100644
--- a/apps/server/src/assets/translations/en/server.json
+++ b/apps/server/src/assets/translations/en/server.json
@@ -567,6 +567,7 @@
"note-not-found": "The meeting note no longer exists.",
"model-missing": "No speech recognition model was found in {{- directory}}.",
"start-failed": "The transcription could not start: {{- message}}",
+ "stopped-unexpectedly": "The transcription stopped unexpectedly. What was transcribed until then is kept.",
"attachment-title": "Transcript"
}
}
diff --git a/packages/commons/src/lib/electron_api_interface.ts b/packages/commons/src/lib/electron_api_interface.ts
index 80e6fad3af..7b762fa23b 100644
--- a/packages/commons/src/lib/electron_api_interface.ts
+++ b/packages/commons/src/lib/electron_api_interface.ts
@@ -724,6 +724,11 @@ export interface TranscriptionState {
noteId: string | null;
/** The sources heard speaking, or whose speech is still being transcribed. */
pendingSources: TranscriptSource[];
+ /**
+ * Why the recording ended without being stopped, sent once with the `idle` state that
+ * follows; `getState()` never returns it.
+ */
+ error?: string;
}
/**
diff --git a/scripts/build-utils.ts b/scripts/build-utils.ts
index 027970b582..f3ae6f2833 100644
--- a/scripts/build-utils.ts
+++ b/scripts/build-utils.ts
@@ -92,7 +92,9 @@ export default class BuildHelper {
// The Codex ACP adapter is copied to RESOURCE_DIR/codex-acp.mjs by the
// build; codex_binary.ts only require.resolve()s the package when running
// from source.
- "@agentclientprotocol/codex-acp"
+ "@agentclientprotocol/codex-acp",
+ // Loads its native addon from its own platform package at runtime.
+ "sherpa-onnx-node"
],
metafile: true,
loader: {