Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(desktop): keep the live transcript and engine settings with the audio
Elian Dorane30dc75a18d1
diff --git a/apps/desktop/src/services/transcription/audio_recorder.spec.ts b/apps/desktop/src/services/transcription/audio_recorder.spec.tsindex fc978304e0..907d094027 100644--- a/apps/desktop/src/services/transcription/audio_recorder.spec.ts+++ b/apps/desktop/src/services/transcription/audio_recorder.spec.ts@@ -3,7 +3,10 @@ import os from "os";import path from "path";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { encodeWavHeader, type RecordingManifest, toPcm16, WavRecorder } from "./audio_recorder";+import {+ encodeWavHeader, getRecordingDirectory, type RecordingManifest, toPcm16, TranscriptFileSink,+ WavRecorder+} from "./audio_recorder";const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));vi.mock("@triliumnext/core", () => ({ getLog: () => log }));@@ -12,48 +15,53 @@ vi.mock("@triliumnext/server/src/services/data_dir.js", () => ({}));const STARTED_AT = Date.parse("2026-10-14T09:30:00.000Z");+const SETTINGS = { recognizer: { decodingMethod: "greedy_search" } };let parentDirectory = "";+/** The recording's own directory, which the recorder creates. */+let directory = "";beforeEach(() => {vi.clearAllMocks();parentDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "trilium-recordings-"));+ directory = path.join(parentDirectory, "recording");});afterEach(() => {fs.rmSync(parentDirectory, { recursive: true, force: true });});-function readRecording() {- const [ name ] = fs.readdirSync(parentDirectory);- const directory = path.join(parentDirectory, name);- const manifest = JSON.parse(- fs.readFileSync(path.join(directory, "recording.json"), "utf8")- ) as RecordingManifest;- return { name, directory, manifest };+function readJson<T>(fileName: string) {+ return JSON.parse(fs.readFileSync(path.join(directory, fileName), "utf8")) as T;}+it("names a recording's directory after when it was created and its note", () => {+ expect(path.basename(getRecordingDirectory(STARTED_AT, "meeting")))+ .toBe("2026-10-14T09-30-00-000Z_meeting");+ expect(path.dirname(getRecordingDirectory(STARTED_AT, "meeting")))+ .toBe(path.join("/data", "transcription-recordings"));+});+describe("WavRecorder", () => {it("keeps each source in a WAV file, lined up by the manifest", () => {- const recorder = new WavRecorder(parentDirectory, "meeting");+ const recorder = new WavRecorder(directory, "meeting", SETTINGS);recorder.open(STARTED_AT);recorder.write("microphone", new Float32Array([ 0.5, -0.5 ]), 1500);recorder.write("system", new Float32Array([ 1 ]), 3000);recorder.write("microphone", new Float32Array([ 2, -2 ]), 1500);// The header already counts every sample before the recording closes.- const { name, directory, manifest } = readRecording();const microphone = fs.readFileSync(path.join(directory, "microphone.wav"));expect(microphone.subarray(0, 44)).toEqual(encodeWavHeader(8));expect(microphone.subarray(44)).toEqual(toPcm16(new Float32Array([ 0.5, -0.5, 1, -1 ])));recorder.close();- expect(name).toBe("2026-10-14T09-30-00-000Z_meeting");expect(fs.readFileSync(path.join(directory, "system.wav")).length).toBe(46);- expect(manifest).toEqual({+ expect(readJson<RecordingManifest>("recording.json")).toEqual({noteId: "meeting",startedAt: "2026-10-14T09:30:00.000Z",sampleRate: 16000,+ engine: SETTINGS,sources: {microphone: { file: "microphone.wav", firstSampleOffsetMs: 1500 },system: { file: "system.wav", firstSampleOffsetMs: 3000 }@@ -62,9 +70,8 @@ describe("WavRecorder", () => {});it("stops keeping the audio after a failed write, logging it once", () => {- const recorder = new WavRecorder(parentDirectory, "meeting");+ const recorder = new WavRecorder(directory, "meeting", SETTINGS);recorder.open(STARTED_AT);- const { directory } = readRecording();fs.rmSync(directory, { recursive: true });recorder.write("microphone", new Float32Array([ 0.5 ]), 0);@@ -76,6 +83,25 @@ describe("WavRecorder", () => {});});+describe("TranscriptFileSink", () => {+ it("rewrites the transcript after every utterance, in the attachment's format", () => {+ const sink = new TranscriptFileSink(directory);+ expect(() => sink.write({ source: "system", offsetMs: 0, text: "Early." }))+ .toThrow("The transcript file was not opened.");++ sink.open(STARTED_AT);+ expect(readJson("transcript.json"))+ .toEqual({ version: 1, startedAt: "2026-10-14T09:30:00.000Z", utterances: [] });++ const utterance =+ { source: "microphone", offsetMs: 1500, durationMs: 800, text: "Hi." } as const;+ sink.write(utterance);+ expect(readJson("transcript.json")).toEqual({+ version: 1, startedAt: "2026-10-14T09:30:00.000Z", utterances: [ utterance ]+ });+ });+});+describe("WAV encoding", () => {it("describes mono 16-bit PCM at 16 kHz", () => {const header = encodeWavHeader(32000);diff --git a/apps/desktop/src/services/transcription/audio_recorder.ts b/apps/desktop/src/services/transcription/audio_recorder.tsindex f7b4513eba..ea4de9b2bd 100644--- a/apps/desktop/src/services/transcription/audio_recorder.ts+++ b/apps/desktop/src/services/transcription/audio_recorder.ts@@ -1,21 +1,32 @@-import type { TranscriptSource } from "@triliumnext/commons";+import type { Transcript, TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";import { getLog } from "@triliumnext/core";import dataDirs from "@triliumnext/server/src/services/data_dir.js";import fs from "fs";import path from "path";import { SAMPLE_RATE } from "./engine";-import type { AudioTap } from "./session";+import type { AudioTap, TranscriptSink } from "./session";/** Where development builds keep the audio of each recording, one directory per recording. */export const RECORDINGS_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-recordings");+/** The transcript the recording produced live, beside the audio it came from. */+export const TRANSCRIPT_FILE = "transcript.json";++/** The directory under {@link RECORDINGS_DIR} for a recording of `noteId` created at `createdAt`. */+export function getRecordingDirectory(createdAt: number, noteId: string) {+ const timestamp = new Date(createdAt).toISOString().replace(/[:.]/g, "-");+ return path.join(RECORDINGS_DIR, `${timestamp}_${noteId}`);+}+/** The `recording.json` next to the audio files, which lines the sources up with the transcript. */export interface RecordingManifest {noteId: string;/** When the recording started, as an ISO 8601 timestamp, like the transcript's `startedAt`. */startedAt: string;sampleRate: number;+ /** The settings of the engine that produced `transcript.json`. */+ engine: Record<string, unknown>;sources: Partial<Record<TranscriptSource, {file: string;/** Milliseconds from `startedAt` to the file's first sample. */@@ -24,39 +35,37 @@ export interface RecordingManifest {}/**- * Keeps a recording's audio as one mono 16-bit WAV file per source, plus a `recording.json`- * manifest, in a directory of its own under `parentDirectory`. Each write also updates the WAV- * header, so the files stay playable if the application quits mid-recording.+ * Keeps a recording's audio in `directory` as one mono 16-bit WAV file per source, plus a+ * `recording.json` manifest. Each write also updates the WAV header, so the files stay playable if+ * the application quits mid-recording.*/export class WavRecorder implements AudioTap {- private directory: string | null = null;private manifest: RecordingManifest | null = null;private readonly files = new Map<TranscriptSource, WavFile>();private failed = false;- constructor(private readonly parentDirectory: string, private readonly noteId: string) {}+ constructor(+ private readonly directory: string,+ private readonly noteId: string,+ private readonly engineSettings: Record<string, unknown>+ ) {}open(startedAt: number) {- const startedAtIso = new Date(startedAt).toISOString();- const directory = path.join(- this.parentDirectory,- `${startedAtIso.replace(/[:.]/g, "-")}_${this.noteId}`- );- fs.mkdirSync(directory, { recursive: true });- this.directory = directory;+ fs.mkdirSync(this.directory, { recursive: true });this.manifest = {noteId: this.noteId,- startedAt: startedAtIso,+ startedAt: new Date(startedAt).toISOString(),sampleRate: SAMPLE_RATE,+ engine: this.engineSettings,sources: {}};this.writeManifest();- getLog().info(`Transcription: keeping the audio in ${directory}`);+ getLog().info(`Transcription: keeping the audio in ${this.directory}`);}write(source: TranscriptSource, samples: Float32Array, firstSampleOffsetMs: number) {const { directory, manifest } = this;- if (this.failed || !directory || !manifest) {+ if (this.failed || !manifest) {return;}@@ -85,10 +94,41 @@ export class WavRecorder implements AudioTap {}private writeManifest() {- if (this.directory && this.manifest) {- const manifestPath = path.join(this.directory, "recording.json");- fs.writeFileSync(manifestPath, JSON.stringify(this.manifest, null, 2));+ const manifestPath = path.join(this.directory, "recording.json");+ fs.writeFileSync(manifestPath, JSON.stringify(this.manifest, null, 2));+ }+}++/**+ * Writes the transcript to {@link TRANSCRIPT_FILE} in `directory` after every utterance, in the+ * format of the `transcript` attachment, so it stays as the recording produced it.+ */+export class TranscriptFileSink implements TranscriptSink {+ private transcript: Transcript | null = null;++ constructor(private readonly directory: string) {}++ open(startedAt: number) {+ fs.mkdirSync(this.directory, { recursive: true });+ this.transcript = {+ version: 1,+ startedAt: new Date(startedAt).toISOString(),+ utterances: []+ };+ this.save();+ }++ write(utterance: TranscriptUtterance) {+ if (!this.transcript) {+ throw new Error("The transcript file was not opened.");}+ this.transcript.utterances.push(utterance);+ this.save();+ }++ private save() {+ const filePath = path.join(this.directory, TRANSCRIPT_FILE);+ fs.writeFileSync(filePath, JSON.stringify(this.transcript, null, 2));}}diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.tsindex 98bf92709f..4d49bf0b2f 100644--- a/apps/desktop/src/services/transcription/engine.ts+++ b/apps/desktop/src/services/transcription/engine.ts@@ -30,6 +30,8 @@ export interface SpeechSegmenter {export interface SpeechEngine {recognizer: SpeechRecognizer;createSegmenter(): SpeechSegmenter;+ /** The settings that shape the transcript, kept with a recording to compare engines by. */+ settings: Record<string, unknown>;}/** Where the models live: Parakeet TDT 0.6B v3 (int8) plus `silero_vad.onnx`. */@@ -38,6 +40,24 @@ export const MODEL_DIR = path.join(dataDirs.TRILIUM_DATA_DIR, "transcription-mod/** Silero VAD consumes 512-sample windows at 16 kHz. */const VAD_WINDOW_SIZE = 512;+const RECOGNIZER_SETTINGS = {+ modelType: "nemo_transducer",+ numThreads: 2,+ provider: "cpu"+};++const DECODING_METHOD = "greedy_search";++const VAD_SETTINGS = {+ threshold: 0.5,+ minSilenceDuration: 0.5,+ minSpeechDuration: 0.25,+ // Past this length, sherpa ends the segment at the next 0.1 s pause, so continuous speech is+ // committed within about 10 s.+ maxSpeechDuration: 8,+ windowSize: VAD_WINDOW_SIZE+};+/*** Loads the local engine: Silero VAD for segmentation and Parakeet for recognition, both through* sherpa-onnx. Recognition runs on the libuv thread pool, so it never blocks the main process.@@ -58,11 +78,9 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {joiner: path.join(MODEL_DIR, "joiner.int8.onnx")},tokens: path.join(MODEL_DIR, "tokens.txt"),- modelType: "nemo_transducer",- numThreads: 2,- provider: "cpu"+ ...RECOGNIZER_SETTINGS},- decodingMethod: "greedy_search"+ decodingMethod: DECODING_METHOD});return {@@ -76,18 +94,17 @@ export async function loadSherpaEngine(): Promise<SpeechEngine> {createSegmenter: () => new VadSegmenter(new sherpa.Vad({sileroVad: {model: path.join(MODEL_DIR, "silero_vad.onnx"),- threshold: 0.5,- minSilenceDuration: 0.5,- minSpeechDuration: 0.25,- // Past this length, sherpa ends the segment at the next 0.1 s pause, so- // continuous speech is committed within about 10 s.- maxSpeechDuration: 8,- windowSize: VAD_WINDOW_SIZE+ ...VAD_SETTINGS},sampleRate: SAMPLE_RATE,numThreads: 1,provider: "cpu"- }, 60))+ }, 60)),+ settings: {+ modelDirectory: MODEL_DIR,+ recognizer: { ...RECOGNIZER_SETTINGS, decodingMethod: DECODING_METHOD },+ vad: VAD_SETTINGS+ }};}diff --git a/apps/desktop/src/services/transcription/index.spec.ts b/apps/desktop/src/services/transcription/index.spec.tsindex 13048e7c7e..9cd5bfdc2a 100644--- a/apps/desktop/src/services/transcription/index.spec.ts+++ b/apps/desktop/src/services/transcription/index.spec.ts@@ -7,6 +7,7 @@ const h = vi.hoisted(() => ({handlers: new Map<string, (...args: unknown[]) => unknown>(),broadcast: [] as TranscriptionState[],notes: new Set<string>([ "meeting" ]),+ isDev: false,loadEngine: vi.fn(),sessionOptions: null as RecordingSessionOptions | null,sessionStart: vi.fn(),@@ -33,7 +34,7 @@ vi.mock("electron", () => ({vi.mock("@triliumnext/core", () => ({becca: { getNote: (noteId: string) => h.notes.has(noteId) ? { noteId } : null },getLog: () => ({ info: vi.fn(), error: vi.fn() }),- utils: { isDev: () => false }+ utils: { isDev: () => h.isDev }}));vi.mock("i18next", () => ({t: (key: string, options?: object) => options ? `${key} ${JSON.stringify(options)}` : key@@ -47,6 +48,17 @@ vi.mock("./engine", async (importOriginal) => ({loadSherpaEngine: h.loadEngine}));vi.mock("./attachment_sink", () => ({ AttachmentSink: class {} }));+vi.mock("./audio_recorder", () => ({+ getRecordingDirectory: (createdAt: number, noteId: string) => `/recordings/${noteId}`,+ TranscriptFileSink: class {+ constructor(readonly directory: string) {}+ },+ WavRecorder: class {+ constructor(+ readonly directory: string, readonly noteId: string, readonly settings: object+ ) {}+ }+}));vi.mock("./session", () => ({RecordingSession: class {start = h.sessionStart;@@ -73,6 +85,7 @@ beforeEach(() => {vi.clearAllMocks();h.handlers.clear();h.broadcast.length = 0;+ h.isDev = false;h.loadEngine.mockResolvedValue({});h.sessionStart.mockResolvedValue([ "microphone", "system" ]);});@@ -112,6 +125,24 @@ describe("transcription IPC", () => {expect(h.broadcast.at(-1)).toEqual(IDLE);});+ it("keeps the audio and the transcript as produced only in development builds", async () => {+ const settings = { recognizer: { decodingMethod: "greedy_search" } };+ h.loadEngine.mockResolvedValue({ settings });+ const invoke = await loadService();++ await invoke("transcription-start", "meeting");+ expect(h.sessionOptions?.sinks).toHaveLength(1);+ expect(h.sessionOptions?.taps).toEqual([]);+ await invoke("transcription-stop");++ h.isDev = true;+ await invoke("transcription-start", "meeting");+ expect(h.sessionOptions?.sinks[1]).toEqual({ directory: "/recordings/meeting" });+ expect(h.sessionOptions?.taps).toEqual([+ { directory: "/recordings/meeting", noteId: "meeting", settings }+ ]);+ });+it("refuses a note that does not exist, without changing state", async () => {const invoke = await loadService();diff --git a/apps/desktop/src/services/transcription/index.ts b/apps/desktop/src/services/transcription/index.tsindex ea053c256d..c6c77987c6 100644--- a/apps/desktop/src/services/transcription/index.ts+++ b/apps/desktop/src/services/transcription/index.ts@@ -4,9 +4,9 @@ import { BrowserWindow, ipcMain } from "electron";import { t } from "i18next";import { AttachmentSink } from "./attachment_sink";-import { RECORDINGS_DIR, WavRecorder } from "./audio_recorder";+import { getRecordingDirectory, TranscriptFileSink, WavRecorder } from "./audio_recorder";import { ElectronAudioCapture } from "./capture";-import { loadSherpaEngine, ModelMissingError } from "./engine";+import { loadSherpaEngine, ModelMissingError, type SpeechEngine } from "./engine";import { type AudioTap, RecordingSession, type TranscriptSink } from "./session";const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };@@ -35,11 +35,11 @@ export async function startRecording(noteId: unknown): Promise<string> {setState({ status: "starting", noteId, pendingSources: [] });try {+ const engine = await loadSherpaEngine();const session = new RecordingSession({capture: new ElectronAudioCapture(),- engine: await loadSherpaEngine(),- sinks: createSinks(noteId),- taps: createTaps(noteId),+ engine,+ ...createReceivers(noteId, engine),onPendingChanged: (pendingSources) => setState({ ...state, pendingSources })});const sources = await session.start();@@ -77,17 +77,19 @@ export function getState() {return state;}-/** Where the utterances of a recording for `noteId` go. */-function createSinks(noteId: string): TranscriptSink[] {- return [ new AttachmentSink(noteId) ];-}-/**- * What else receives the audio of a recording for `noteId`: development builds keep it, to- * transcribe it again while the engine is tuned.+ * Where the utterances and the audio of a recording for `noteId` go. Development builds also keep+ * the audio and the transcript as produced, to transcribe the audio again while the engine is tuned.*/-function createTaps(noteId: string): AudioTap[] {- return coreUtils.isDev() ? [ new WavRecorder(RECORDINGS_DIR, noteId) ] : [];+function createReceivers(noteId: string, engine: SpeechEngine) {+ const sinks: TranscriptSink[] = [ new AttachmentSink(noteId) ];+ const taps: AudioTap[] = [];+ if (coreUtils.isDev()) {+ const directory = getRecordingDirectory(Date.now(), noteId);+ sinks.push(new TranscriptFileSink(directory));+ taps.push(new WavRecorder(directory, noteId, engine.settings));+ }+ return { sinks, taps };}function setState(newState: TranscriptionState) {diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.tsindex c8bfa91d00..b6292b5db6 100644--- a/apps/desktop/src/services/transcription/session.spec.ts+++ b/apps/desktop/src/services/transcription/session.spec.ts@@ -44,6 +44,7 @@ function createHarness(extraSinks: TranscriptSink[] = [], taps: AudioTap[] = [])};const flushed: SpeechSegment[][] = [];const engine = {+ settings: {},recognizer: {transcribe: vi.fn(async (samples: Float32Array) =>samples[0] ? `utterance ${samples[0]}` : "")