Skip to content

Build your personal knowledge base with Trilium Notes

feat(desktop): keep the live transcript and engine settings with the audio

Elian Dorane30dc75a18d1
diff --git a/apps/desktop/src/services/transcription/audio_recorder.spec.ts b/apps/desktop/src/services/transcription/audio_recorder.spec.ts
index fc978304e0..907d094027 100644
--- a/apps/desktop/src/services/transcription/audio_recorder.spec.ts
+++ b/apps/desktop/src/services/transcription/audio_recorder.spec.ts
@@ -3,7 +3,10 @@ import os from "os";
import path from "path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { encodeWavHeader, type RecordingManifest, toPcm16, WavRecorder } from "./audio_recorder";
+import {
+ encodeWavHeader, getRecordingDirectory, type RecordingManifest, toPcm16, TranscriptFileSink,
+ WavRecorder
+} from "./audio_recorder";
const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));
vi.mock("@triliumnext/core", () => ({ getLog: () => log }));
@@ -12,48 +15,53 @@ vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({
}));
const STARTED_AT = Date.parse("2026-10-14T09:30:00.000Z");
+const SETTINGS = { recognizer: { decodingMethod: "greedy_search" } };
let parentDirectory = "";
+/** The recording's own directory, which the recorder creates. */
+let directory = "";
beforeEach(() => {
vi.clearAllMocks();
parentDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "trilium-recordings-"));
+ directory = path.join(parentDirectory, "recording");
});
afterEach(() => {
fs.rmSync(parentDirectory, { recursive: true, force: true });
});
-function readRecording() {
- const [ name ] = fs.readdirSync(parentDirectory);
- const directory = path.join(parentDirectory, name);
- const manifest = JSON.parse(
- fs.readFileSync(path.join(directory, "recording.json"), "utf8")
- ) as RecordingManifest;
- return { name, directory, manifest };
+function readJson<T>(fileName: string) {
+ return JSON.parse(fs.readFileSync(path.join(directory, fileName), "utf8")) as T;
}
+it("names a recording's directory after when it was created and its note", () => {
+ expect(path.basename(getRecordingDirectory(STARTED_AT, "meeting")))
+ .toBe("2026-10-14T09-30-00-000Z_meeting");
+ expect(path.dirname(getRecordingDirectory(STARTED_AT, "meeting")))
+ .toBe(path.join("/data", "transcription-recordings"));
+});
+
describe("WavRecorder", () => {
it("keeps each source in a WAV file, lined up by the manifest", () => {
- const recorder = new WavRecorder(parentDirectory, "meeting");
+ const recorder = new WavRecorder(directory, "meeting", SETTINGS);
recorder.open(STARTED_AT);
recorder.write("microphone", new Float32Array([ 0.5, -0.5 ]), 1500);
recorder.write("system", new Float32Array([ 1 ]), 3000);
recorder.write("microphone", new Float32Array([ 2, -2 ]), 1500);
// The header already counts every sample before the recording closes.
- const { name, directory, manifest } = readRecording();
const microphone = fs.readFileSync(path.join(directory, "microphone.wav"));
expect(microphone.subarray(0, 44)).toEqual(encodeWavHeader(8));
expect(microphone.subarray(44)).toEqual(toPcm16(new Float32Array([ 0.5, -0.5, 1, -1 ])));
recorder.close();
- expect(name).toBe("2026-10-14T09-30-00-000Z_meeting");
expect(fs.readFileSync(path.join(directory, "system.wav")).length).toBe(46);
- expect(manifest).toEqual({
+ expect(readJson<RecordingManifest>("recording.json")).toEqual({
noteId: "meeting",
startedAt: "2026-10-14T09:30:00.000Z",
sampleRate: 16000,
+ engine: SETTINGS,
sources: {
microphone: { file: "microphone.wav", firstSampleOffsetMs: 1500 },
system: { file: "system.wav", firstSampleOffsetMs: 3000 }
@@ -62,9 +70,8 @@ describe("WavRecorder", () => {
});
it("stops keeping the audio after a failed write, logging it once", () => {
- const recorder = new WavRecorder(parentDirectory, "meeting");
+ const recorder = new WavRecorder(directory, "meeting", SETTINGS);
recorder.open(STARTED_AT);
- const { directory } = readRecording();
fs.rmSync(directory, { recursive: true });
recorder.write("microphone", new Float32Array([ 0.5 ]), 0);
@@ -76,6 +83,25 @@ describe("WavRecorder", () => {
});
});
+describe("TranscriptFileSink", () => {
+ it("rewrites the transcript after every utterance, in the attachment's format", () => {
+ const sink = new TranscriptFileSink(directory);
+ expect(() => sink.write({ source: "system", offsetMs: 0, text: "Early." }))
+ .toThrow("The transcript file was not opened.");
+
+ sink.open(STARTED_AT);
+ expect(readJson("transcript.json"))
+ .toEqual({ version: 1, startedAt: "2026-10-14T09:30:00.000Z", utterances: [] });
+
+ const utterance =
+ { source: "microphone", offsetMs: 1500, durationMs: 800, text: "Hi." } as const;
+ sink.write(utterance);
+ expect(readJson("transcript.json")).toEqual({
+ version: 1, startedAt: "2026-10-14T09:30:00.000Z", utterances: [ utterance ]
+ });
+ });
+});
+
describe("WAV encoding", () => {
it("describes mono 16-bit PCM at 16 kHz", () => {
const header = encodeWavHeader(32000);
diff --git a/apps/desktop/src/services/transcription/audio_recorder.ts b/apps/desktop/src/services/transcription/audio_recorder.ts
index f7b4513eba..ea4de9b2bd 100644
--- a/apps/desktop/src/services/transcription/audio_recorder.ts
+++ b/apps/desktop/src/services/transcription/audio_recorder.ts
@@ -1,21 +1,32 @@
-import type { TranscriptSource } from "@triliumnext/commons";
+import type { Transcript, TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
import { getLog } from "@triliumnext/core";
import dataDirs from "@triliumnext/server/src/services/data_dir.js";
import fs from "fs";
import path from "path";
import { SAMPLE_RATE } from "./engine";
-import type { AudioTap } from "./session";
+import type { AudioTap, TranscriptSink } from "./session";
/** Where development builds keep the audio of each recording, one directory per recording. */
export const RECORDINGS_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-recordings");
+/** The transcript the recording produced live, beside the audio it came from. */
+export const TRANSCRIPT_FILE = "transcript.json";
+
+/** The directory under {@link RECORDINGS_DIR} for a recording of `noteId` created at `createdAt`. */
+export function getRecordingDirectory(createdAt: number, noteId: string) {
+ const timestamp = new Date(createdAt).toISOString().replace(/[:.]/g, "-");
+ return path.join(RECORDINGS_DIR, `${timestamp}_${noteId}`);
+}
+
/** The `recording.json` next to the audio files, which lines the sources up with the transcript. */
export interface RecordingManifest {
noteId: string;
/** When the recording started, as an ISO 8601 timestamp, like the transcript's `startedAt`. */
startedAt: string;
sampleRate: number;
+ /** The settings of the engine that produced `transcript.json`. */
+ engine: Record<string, unknown>;
sources: Partial<Record<TranscriptSource, {
file: string;
/** Milliseconds from `startedAt` to the file's first sample. */
@@ -24,39 +35,37 @@ export interface RecordingManifest {
}
/**
- * Keeps a recording's audio as one mono 16-bit WAV file per source, plus a `recording.json`
- * manifest, in a directory of its own under `parentDirectory`. Each write also updates the WAV
- * header, so the files stay playable if the application quits mid-recording.
+ * Keeps a recording's audio in `directory` as one mono 16-bit WAV file per source, plus a
+ * `recording.json` manifest. Each write also updates the WAV header, so the files stay playable if
+ * the application quits mid-recording.
*/
export class WavRecorder implements AudioTap {
- private directory: string | null = null;
private manifest: RecordingManifest | null = null;
private readonly files = new Map<TranscriptSource, WavFile>();
private failed = false;
- constructor(private readonly parentDirectory: string, private readonly noteId: string) {}
+ constructor(
+ private readonly directory: string,
+ private readonly noteId: string,
+ private readonly engineSettings: Record<string, unknown>
+ ) {}
open(startedAt: number) {
- const startedAtIso = new Date(startedAt).toISOString();
- const directory = path.join(
- this.parentDirectory,
- `${startedAtIso.replace(/[:.]/g, "-")}_${this.noteId}`
- );
- fs.mkdirSync(directory, { recursive: true });
- this.directory = directory;
+ fs.mkdirSync(this.directory, { recursive: true });
this.manifest = {
noteId: this.noteId,
- startedAt: startedAtIso,
+ startedAt: new Date(startedAt).toISOString(),
sampleRate: SAMPLE_RATE,
+ engine: this.engineSettings,
sources: {}
};
this.writeManifest();
- getLog().info(`Transcription: keeping the audio in ${directory}`);
+ getLog().info(`Transcription: keeping the audio in ${this.directory}`);
}
write(source: TranscriptSource, samples: Float32Array, firstSampleOffsetMs: number) {
const { directory, manifest } = this;
- if (this.failed || !directory || !manifest) {
+ if (this.failed || !manifest) {
return;
}
@@ -85,10 +94,41 @@ export class WavRecorder implements AudioTap {
}
private writeManifest() {
- if (this.directory && this.manifest) {
- const manifestPath = path.join(this.directory, "recording.json");
- fs.writeFileSync(manifestPath, JSON.stringify(this.manifest, null, 2));
+ const manifestPath = path.join(this.directory, "recording.json");
+ fs.writeFileSync(manifestPath, JSON.stringify(this.manifest, null, 2));
+ }
+}
+
+/**
+ * Writes the transcript to {@link TRANSCRIPT_FILE} in `directory` after every utterance, in the
+ * format of the `transcript` attachment, so it stays as the recording produced it.
+ */
+export class TranscriptFileSink implements TranscriptSink {
+ private transcript: Transcript | null = null;
+
+ constructor(private readonly directory: string) {}
+
+ open(startedAt: number) {
+ fs.mkdirSync(this.directory, { recursive: true });
+ this.transcript = {
+ version: 1,
+ startedAt: new Date(startedAt).toISOString(),
+ utterances: []
+ };
+ this.save();
+ }
+
+ write(utterance: TranscriptUtterance) {
+ if (!this.transcript) {
+ throw new Error("The transcript file was not opened.");
}
+ this.transcript.utterances.push(utterance);
+ this.save();
+ }
+
+ private save() {
+ const filePath = path.join(this.directory, TRANSCRIPT_FILE);
+ fs.writeFileSync(filePath, JSON.stringify(this.transcript, null, 2));
}
}
diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.ts
index 98bf92709f..4d49bf0b2f 100644
--- a/apps/desktop/src/services/transcription/engine.ts
+++ b/apps/desktop/src/services/transcription/engine.ts
@@ -30,6 +30,8 @@ export interface SpeechSegmenter {
export interface SpeechEngine {
recognizer: SpeechRecognizer;
createSegmenter(): SpeechSegmenter;
+ /** The settings that shape the transcript, kept with a recording to compare engines by. */
+ settings: Record<string, unknown>;
}
/** Where the models live: Parakeet TDT 0.6B v3 (int8) plus `silero_vad.onnx`. */
@@ -38,6 +40,24 @@ export const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-mod
/** Silero VAD consumes 512-sample windows at 16 kHz. */
const VAD_WINDOW_SIZE = 512;
+const RECOGNIZER_SETTINGS = {
+ modelType: "nemo_transducer",
+ numThreads: 2,
+ provider: "cpu"
+};
+
+const DECODING_METHOD = "greedy_search";
+
+const VAD_SETTINGS = {
+ threshold: 0.5,
+ minSilenceDuration: 0.5,
+ minSpeechDuration: 0.25,
+ // Past this length, sherpa ends the segment at the next 0.1 s pause, so continuous speech is
+ // committed within about 10 s.
+ maxSpeechDuration: 8,
+ windowSize: VAD_WINDOW_SIZE
+};
+
/**
* Loads the local engine: Silero VAD for segmentation and Parakeet for recognition, both through
* sherpa-onnx. Recognition runs on the libuv thread pool, so it never blocks the main process.
@@ -58,11 +78,9 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {
joiner: path.join(MODEL_DIR, "joiner.int8.onnx")
},
tokens: path.join(MODEL_DIR, "tokens.txt"),
- modelType: "nemo_transducer",
- numThreads: 2,
- provider: "cpu"
+ ...RECOGNIZER_SETTINGS
},
- decodingMethod: "greedy_search"
+ decodingMethod: DECODING_METHOD
});
return {
@@ -76,18 +94,17 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {
createSegmenter: () => new VadSegmenter(new sherpa.Vad({
sileroVad: {
model: path.join(MODEL_DIR, "silero_vad.onnx"),
- threshold: 0.5,
- minSilenceDuration: 0.5,
- minSpeechDuration: 0.25,
- // Past this length, sherpa ends the segment at the next 0.1 s pause, so
- // continuous speech is committed within about 10 s.
- maxSpeechDuration: 8,
- windowSize: VAD_WINDOW_SIZE
+ ...VAD_SETTINGS
},
sampleRate: SAMPLE_RATE,
numThreads: 1,
provider: "cpu"
- }, 60))
+ }, 60)),
+ settings: {
+ modelDirectory: MODEL_DIR,
+ recognizer: { ...RECOGNIZER_SETTINGS, decodingMethod: DECODING_METHOD },
+ vad: VAD_SETTINGS
+ }
};
}
diff --git a/apps/desktop/src/services/transcription/index.spec.ts b/apps/desktop/src/services/transcription/index.spec.ts
index 13048e7c7e..9cd5bfdc2a 100644
--- a/apps/desktop/src/services/transcription/index.spec.ts
+++ b/apps/desktop/src/services/transcription/index.spec.ts
@@ -7,6 +7,7 @@ const h = vi.hoisted(() => ({
handlers: new Map<string, (...args: unknown[]) => unknown>(),
broadcast: [] as TranscriptionState[],
notes: new Set<string>([ "meeting" ]),
+ isDev: false,
loadEngine: vi.fn(),
sessionOptions: null as RecordingSessionOptions | null,
sessionStart: vi.fn(),
@@ -33,7 +34,7 @@ vi.mock("electron", () => ({
vi.mock("@triliumnext/core", () => ({
becca: { getNote: (noteId: string) => h.notes.has(noteId) ? { noteId } : null },
getLog: () => ({ info: vi.fn(), error: vi.fn() }),
- utils: { isDev: () => false }
+ utils: { isDev: () => h.isDev }
}));
vi.mock("i18next", () => ({
t: (key: string, options?: object) => options ? `${key} ${JSON.stringify(options)}` : key
@@ -47,6 +48,17 @@ vi.mock("./engine", async (importOriginal) => ({
loadSherpaEngine: h.loadEngine
}));
vi.mock("./attachment_sink", () => ({ AttachmentSink: class {} }));
+vi.mock("./audio_recorder", () => ({
+ getRecordingDirectory: (createdAt: number, noteId: string) => `/recordings/${noteId}`,
+ TranscriptFileSink: class {
+ constructor(readonly directory: string) {}
+ },
+ WavRecorder: class {
+ constructor(
+ readonly directory: string, readonly noteId: string, readonly settings: object
+ ) {}
+ }
+}));
vi.mock("./session", () => ({
RecordingSession: class {
start = h.sessionStart;
@@ -73,6 +85,7 @@ beforeEach(() => {
vi.clearAllMocks();
h.handlers.clear();
h.broadcast.length = 0;
+ h.isDev = false;
h.loadEngine.mockResolvedValue({});
h.sessionStart.mockResolvedValue([ "microphone", "system" ]);
});
@@ -112,6 +125,24 @@ describe("transcription IPC", () => {
expect(h.broadcast.at(-1)).toEqual(IDLE);
});
+ it("keeps the audio and the transcript as produced only in development builds", async () => {
+ const settings = { recognizer: { decodingMethod: "greedy_search" } };
+ h.loadEngine.mockResolvedValue({ settings });
+ const invoke = await loadService();
+
+ await invoke("transcription-start", "meeting");
+ expect(h.sessionOptions?.sinks).toHaveLength(1);
+ expect(h.sessionOptions?.taps).toEqual([]);
+ await invoke("transcription-stop");
+
+ h.isDev = true;
+ await invoke("transcription-start", "meeting");
+ expect(h.sessionOptions?.sinks[1]).toEqual({ directory: "/recordings/meeting" });
+ expect(h.sessionOptions?.taps).toEqual([
+ { directory: "/recordings/meeting", noteId: "meeting", settings }
+ ]);
+ });
+
it("refuses a note that does not exist, without changing state", async () => {
const invoke = await loadService();
diff --git a/apps/desktop/src/services/transcription/index.ts b/apps/desktop/src/services/transcription/index.ts
index ea053c256d..c6c77987c6 100644
--- a/apps/desktop/src/services/transcription/index.ts
+++ b/apps/desktop/src/services/transcription/index.ts
@@ -4,9 +4,9 @@ import { BrowserWindow, ipcMain } from "electron";
import { t } from "i18next";
import { AttachmentSink } from "./attachment_sink";
-import { RECORDINGS_DIR, WavRecorder } from "./audio_recorder";
+import { getRecordingDirectory, TranscriptFileSink, WavRecorder } from "./audio_recorder";
import { ElectronAudioCapture } from "./capture";
-import { loadSherpaEngine, ModelMissingError } from "./engine";
+import { loadSherpaEngine, ModelMissingError, type SpeechEngine } from "./engine";
import { type AudioTap, RecordingSession, type TranscriptSink } from "./session";
const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };
@@ -35,11 +35,11 @@ export async function startRecording(noteId: unknown): Promise<string> {
setState({ status: "starting", noteId, pendingSources: [] });
try {
+ const engine = await loadSherpaEngine();
const session = new RecordingSession({
capture: new ElectronAudioCapture(),
- engine: await loadSherpaEngine(),
- sinks: createSinks(noteId),
- taps: createTaps(noteId),
+ engine,
+ ...createReceivers(noteId, engine),
onPendingChanged: (pendingSources) => setState({ ...state, pendingSources })
});
const sources = await session.start();
@@ -77,17 +77,19 @@ export function getState() {
return state;
}
-/** Where the utterances of a recording for `noteId` go. */
-function createSinks(noteId: string): TranscriptSink[] {
- return [ new AttachmentSink(noteId) ];
-}
-
/**
- * What else receives the audio of a recording for `noteId`: development builds keep it, to
- * transcribe it again while the engine is tuned.
+ * Where the utterances and the audio of a recording for `noteId` go. Development builds also keep
+ * the audio and the transcript as produced, to transcribe the audio again while the engine is tuned.
*/
-function createTaps(noteId: string): AudioTap[] {
- return coreUtils.isDev() ? [ new WavRecorder(RECORDINGS_DIR, noteId) ] : [];
+function createReceivers(noteId: string, engine: SpeechEngine) {
+ const sinks: TranscriptSink[] = [ new AttachmentSink(noteId) ];
+ const taps: AudioTap[] = [];
+ if (coreUtils.isDev()) {
+ const directory = getRecordingDirectory(Date.now(), noteId);
+ sinks.push(new TranscriptFileSink(directory));
+ taps.push(new WavRecorder(directory, noteId, engine.settings));
+ }
+ return { sinks, taps };
}
function setState(newState: TranscriptionState) {
diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.ts
index c8bfa91d00..b6292b5db6 100644
--- a/apps/desktop/src/services/transcription/session.spec.ts
+++ b/apps/desktop/src/services/transcription/session.spec.ts
@@ -44,6 +44,7 @@ function createHarness(extraSinks: TranscriptSink[] = [], taps: AudioTap[] = [])
};
const flushed: SpeechSegment[][] = [];
const engine = {
+ settings: {},
recognizer: {
transcribe: vi.fn(async (samples: Float32Array) =>
samples[0] ? `utterance ${samples[0]}` : "")