Skip to content

Build your personal knowledge base with Trilium Notes

feat(desktop): stream the meeting transcript into an attachment

Elian Doran4c2efa1c280a
diff --git a/apps/client/src/services/attachment_groups.ts b/apps/client/src/services/attachment_groups.ts
index 2e04f08896..87ceda32ca 100644
--- a/apps/client/src/services/attachment_groups.ts
+++ b/apps/client/src/services/attachment_groups.ts
@@ -35,7 +35,9 @@ export const ATTACHMENT_ROLE_GROUPS: Record<AttachmentRole, AttachmentGroup> = {
// than attached to the note by hand.
canvasLibraryItem: "system",
// The file an import was read from, kept so an import that went wrong can be looked at again.
- importSource: "system"
+ importSource: "system",
+ // A meeting recording's transcript: made by the app, but what the reader asked for.
+ transcript: "user"
};
/**
diff --git a/apps/client/src/services/attachment_role_names.spec.ts b/apps/client/src/services/attachment_role_names.spec.ts
index 6f36d3e2fd..8f21e545ad 100644
--- a/apps/client/src/services/attachment_role_names.spec.ts
+++ b/apps/client/src/services/attachment_role_names.spec.ts
@@ -10,7 +10,7 @@ const { attachmentRoleLabel } = await import("./attachment_role_names.js");
describe("attachmentRoleLabel", () => {
it("names the app's own doing, which nothing else in the card does", () => {
// "Site icon" is the whole reason that row is not a picture the reader forgot placing.
- for (const role of [ "favicon", "coverImage", "viewConfig", "canvasLibraryItem", "importSource" ]) {
+ for (const role of [ "favicon", "coverImage", "viewConfig", "canvasLibraryItem", "importSource", "transcript" ]) {
expect(attachmentRoleLabel(role), role).toBe(`attachment_roles.${role}`);
}
});
diff --git a/apps/client/src/services/attachment_role_names.ts b/apps/client/src/services/attachment_role_names.ts
index 41d0fada9d..7322473a07 100644
--- a/apps/client/src/services/attachment_role_names.ts
+++ b/apps/client/src/services/attachment_role_names.ts
@@ -24,7 +24,8 @@ const ATTACHMENT_ROLE_NAMES: Record<NamedAttachmentRole, () => string> = {
coverImage: () => t("attachment_roles.coverImage"),
viewConfig: () => t("attachment_roles.viewConfig"),
canvasLibraryItem: () => t("attachment_roles.canvasLibraryItem"),
- importSource: () => t("attachment_roles.importSource")
+ importSource: () => t("attachment_roles.importSource"),
+ transcript: () => t("attachment_roles.transcript")
};
/**
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json
index af734a4189..c463114f7d 100644
--- a/apps/client/src/translations/en/translation.json
+++ b/apps/client/src/translations/en/translation.json
@@ -2443,7 +2443,8 @@
"coverImage": "Cover image",
"viewConfig": "View settings",
"canvasLibraryItem": "Canvas library",
- "importSource": "Import source"
+ "importSource": "Import source",
+ "transcript": "Meeting transcript"
},
"bookmark_switch": {
"bookmark": "Bookmark"
diff --git a/apps/desktop/src/services/transcription/attachment_sink.spec.ts b/apps/desktop/src/services/transcription/attachment_sink.spec.ts
new file mode 100644
index 0000000000..141e7c9c18
--- /dev/null
+++ b/apps/desktop/src/services/transcription/attachment_sink.spec.ts
@@ -0,0 +1,72 @@
+import type { Transcript } from "@triliumnext/commons";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const h = vi.hoisted(() => ({
+ saveAttachment: vi.fn(),
+ setContent: vi.fn(),
+ transactions: 0
+}));
+
+vi.mock("@triliumnext/core", () => ({
+ becca: {
+ getNoteOrThrow: (noteId: string) => ({ noteId, saveAttachment: h.saveAttachment }),
+ getAttachmentOrThrow: (attachmentId: string) => ({ attachmentId, setContent: h.setContent })
+ },
+ cls: { init: <T>(callback: () => T) => callback() },
+ getSql: () => ({
+ transactional: <T>(callback: () => T) => {
+ h.transactions++;
+ return callback();
+ }
+ })
+}));
+vi.mock("i18next", () => ({ t: (key: string) => key }));
+
+const { AttachmentSink } = await import("./attachment_sink");
+
+const STARTED_AT = Date.UTC(2026, 9, 13, 14, 30);
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ h.transactions = 0;
+ h.saveAttachment.mockReturnValue({ attachmentId: "transcript1" });
+});
+
+describe("AttachmentSink", () => {
+ it("creates an empty transcript attachment on the meeting note when it opens", () => {
+ new AttachmentSink("meeting").open(STARTED_AT);
+
+ expect(h.saveAttachment).toHaveBeenCalledExactlyOnceWith({
+ role: "transcript",
+ mime: "application/json",
+ title: "transcription.attachment-title",
+ content: expect.any(String)
+ });
+ const content = JSON.parse(h.saveAttachment.mock.calls[0][0].content) as Transcript;
+ expect(content).toEqual({
+ version: 1, startedAt: "2026-10-13T14:30:00.000Z", utterances: []
+ });
+ });
+
+ it("rewrites the whole transcript after every utterance, in a transaction each", () => {
+ const sink = new AttachmentSink("meeting");
+ sink.open(STARTED_AT);
+ sink.write({ source: "microphone", offsetMs: 1500, text: "Can you hear me?" });
+ sink.write({ source: "system", offsetMs: 3000, text: "Yes." });
+
+ expect(h.setContent).toHaveBeenCalledTimes(2);
+ const last = JSON.parse(h.setContent.mock.calls[1][0]) as Transcript;
+ expect(last.utterances).toEqual([
+ { source: "microphone", offsetMs: 1500, text: "Can you hear me?" },
+ { source: "system", offsetMs: 3000, text: "Yes." }
+ ]);
+ expect(h.transactions).toBe(3);
+ });
+
+ it("refuses to write before it is open", () => {
+ const sink = new AttachmentSink("meeting");
+ expect(() => sink.write({ source: "system", offsetMs: 0, text: "Hi" }))
+ .toThrow("The transcript attachment was not created.");
+ expect(h.setContent).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/desktop/src/services/transcription/attachment_sink.ts b/apps/desktop/src/services/transcription/attachment_sink.ts
new file mode 100644
index 0000000000..98ef507ac6
--- /dev/null
+++ b/apps/desktop/src/services/transcription/attachment_sink.ts
@@ -0,0 +1,50 @@
+import { type Transcript, TRANSCRIPT_MIME, type TranscriptUtterance } from "@triliumnext/commons";
+import { becca, cls, getSql } from "@triliumnext/core";
+import { t } from "i18next";
+
+import type { TranscriptSink } from "./session";
+
+/**
+ * Keeps the transcript in a `transcript` attachment of the meeting note, rewritten after every
+ * utterance. Each write commits its own transaction, which sends the change to every open window.
+ */
+export class AttachmentSink implements TranscriptSink {
+ private transcript: Transcript | null = null;
+ private attachmentId: string | null = null;
+
+ constructor(private readonly noteId: string) {}
+
+ open(startedAt: number) {
+ this.transcript = {
+ version: 1,
+ startedAt: new Date(startedAt).toISOString(),
+ utterances: []
+ };
+ const content = serialize(this.transcript);
+ this.attachmentId = inTransaction(() => becca.getNoteOrThrow(this.noteId).saveAttachment({
+ role: "transcript",
+ mime: TRANSCRIPT_MIME,
+ title: t("transcription.attachment-title"),
+ content
+ }).attachmentId) ?? null;
+ }
+
+ write(utterance: TranscriptUtterance) {
+ const { transcript, attachmentId } = this;
+ if (!transcript || !attachmentId) {
+ throw new Error("The transcript attachment was not created.");
+ }
+
+ transcript.utterances.push(utterance);
+ const content = serialize(transcript);
+ inTransaction(() => becca.getAttachmentOrThrow(attachmentId).setContent(content));
+ }
+}
+
+function serialize(transcript: Transcript) {
+ return JSON.stringify(transcript, null, 2);
+}
+
+function inTransaction<T>(callback: () => T): T {
+ return cls.init(() => getSql().transactional(callback));
+}
diff --git a/apps/desktop/src/services/transcription/capture.ts b/apps/desktop/src/services/transcription/capture.ts
index 0834e99124..fdb7b18b68 100644
--- a/apps/desktop/src/services/transcription/capture.ts
+++ b/apps/desktop/src/services/transcription/capture.ts
@@ -1,3 +1,4 @@
+import type { TranscriptSource } from "@triliumnext/commons";
import { getLog } from "@triliumnext/core";
import { app, BrowserWindow, ipcMain, type IpcMainEvent, session } from "electron";
import fs from "fs";
@@ -5,10 +6,7 @@ import path from "path";
import { SAMPLE_RATE } from "./engine";
-/** The microphone carries the local speaker; the system output carries everyone else. */
-export type AudioSource = "microphone" | "system";
-
-export type SamplesListener = (source: AudioSource, samples: Float32Array) => void;
+export type SamplesListener = (source: TranscriptSource, samples: Float32Array) => void;
/** Produces mono {@link SAMPLE_RATE} samples per source until stopped. */
export interface AudioCapture {
@@ -17,7 +15,7 @@ export interface AudioCapture {
*
* @throws when no source could start.
*/
- start(onSamples: SamplesListener): Promise<AudioSource[]>;
+ start(onSamples: SamplesListener): Promise<TranscriptSource[]>;
stop(): void;
}
@@ -29,7 +27,8 @@ const SOURCE_STARTERS = [
[ "system", "captureSystemAudio()" ]
] as const;
-type SamplesIpcListener = (event: IpcMainEvent, source: AudioSource, samples: Float32Array) => void;
+type SamplesIpcListener =
+ (event: IpcMainEvent, source: TranscriptSource, samples: Float32Array) => void;
/**
* Captures the default microphone and the system audio output in a hidden window, which streams
@@ -50,7 +49,7 @@ export class ElectronAudioCapture implements AudioCapture {
};
ipcMain.on(SAMPLES_CHANNEL, this.listener);
- const sources: AudioSource[] = [];
+ const sources: TranscriptSource[] = [];
for (const [ source, call ] of SOURCE_STARTERS) {
try {
// getDisplayMedia() requires a user gesture, which the hidden window only gets
diff --git a/apps/desktop/src/services/transcription/index.spec.ts b/apps/desktop/src/services/transcription/index.spec.ts
index 08e46b8a9c..7b37bd06d7 100644
--- a/apps/desktop/src/services/transcription/index.spec.ts
+++ b/apps/desktop/src/services/transcription/index.spec.ts
@@ -42,8 +42,8 @@ vi.mock("./engine", async (importOriginal) => ({
...(await importOriginal<typeof import("./engine")>()),
loadSherpaEngine: h.loadEngine
}));
+vi.mock("./attachment_sink", () => ({ AttachmentSink: class {} }));
vi.mock("./session", () => ({
- LogSink: class {},
RecordingSession: class {
start = h.sessionStart;
stop = h.sessionStop;
diff --git a/apps/desktop/src/services/transcription/index.ts b/apps/desktop/src/services/transcription/index.ts
index f2fa140b6c..45fa601dfb 100644
--- a/apps/desktop/src/services/transcription/index.ts
+++ b/apps/desktop/src/services/transcription/index.ts
@@ -3,9 +3,10 @@ import { becca, getLog } from "@triliumnext/core";
import { BrowserWindow, ipcMain } from "electron";
import { t } from "i18next";
+import { AttachmentSink } from "./attachment_sink";
import { ElectronAudioCapture } from "./capture";
import { loadSherpaEngine, ModelMissingError } from "./engine";
-import { LogSink, RecordingSession, type TranscriptSink } from "./session";
+import { RecordingSession, type TranscriptSink } from "./session";
let state: TranscriptionState = { status: "idle", noteId: null };
let activeSession: RecordingSession | null = null;
@@ -61,6 +62,7 @@ export async function stopRecording() {
activeSession = null;
try {
await session.stop();
+ getLog().info(`Transcription: recording for note ${state.noteId} stopped.`);
} finally {
setState({ status: "idle", noteId: null });
}
@@ -72,7 +74,7 @@ export function getState() {
/** Where the utterances of a recording for `noteId` go. */
function createSinks(noteId: string): TranscriptSink[] {
- return [ new LogSink(noteId) ];
+ return [ new AttachmentSink(noteId) ];
}
function setState(newState: TranscriptionState) {
diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.ts
index c851e61ae9..9e32f57034 100644
--- a/apps/desktop/src/services/transcription/session.spec.ts
+++ b/apps/desktop/src/services/transcription/session.spec.ts
@@ -1,8 +1,9 @@
+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
import { beforeEach, describe, expect, it, vi } from "vitest";
-import type { AudioSource, SamplesListener } from "./capture";
+import type { SamplesListener } from "./capture";
import type { SpeechSegment } from "./engine";
-import { RecordingSession, type TranscriptSink, type Utterance } from "./session";
+import { RecordingSession, type TranscriptSink } from "./session";
const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));
vi.mock("@triliumnext/core", () => ({ getLog: () => log }));
@@ -27,7 +28,7 @@ function createHarness(extraSinks: TranscriptSink[] = []) {
const capture = {
start: vi.fn(async (listener: SamplesListener) => {
onSamples = listener;
- return [ "microphone", "system" ] as AudioSource[];
+ return [ "microphone", "system" ] as TranscriptSource[];
}),
stop: vi.fn()
};
@@ -45,9 +46,10 @@ function createHarness(extraSinks: TranscriptSink[] = []) {
};
})
};
- const written: Utterance[] = [];
+ const written: TranscriptUtterance[] = [];
const sink = {
- write: vi.fn((utterance: Utterance) => {
+ open: vi.fn(),
+ write: vi.fn((utterance: TranscriptUtterance) => {
written.push(utterance);
}),
close: vi.fn()
@@ -58,7 +60,7 @@ function createHarness(extraSinks: TranscriptSink[] = []) {
return {
session, capture, engine, sink, written, flushed,
/** Delivers `samples` from `source` when the clock reads `at`. */
- deliver(source: AudioSource, samples: Float32Array, at: number) {
+ deliver(source: TranscriptSource, samples: Float32Array, at: number) {
clock = at;
onSamples(source, samples);
}
@@ -71,8 +73,11 @@ beforeEach(() => {
describe("RecordingSession", () => {
it("labels each source's utterances with their offset from the recording start", async () => {
- const { session, engine, deliver, written } = createHarness();
+ const { session, capture, engine, sink, deliver, written } = createHarness();
await session.start();
+ expect(sink.open).toHaveBeenCalledExactlyOnceWith(1000);
+ expect(sink.open.mock.invocationCallOrder[0])
+ .toBeLessThan(capture.start.mock.invocationCallOrder[0]);
// The microphone's first one-second chunk arrives 2 s in, so it began 1 s in.
deliver("microphone", utteranceChunk(1, 8000), 3000);
@@ -113,6 +118,16 @@ describe("RecordingSession", () => {
expect(log.error).toHaveBeenCalledTimes(2);
});
+ it("does not start capturing when a sink cannot open", async () => {
+ const { session, capture, sink } = createHarness();
+ sink.open.mockImplementationOnce(() => {
+ throw new Error("note deleted");
+ });
+
+ await expect(session.start()).rejects.toThrow("note deleted");
+ expect(capture.start).not.toHaveBeenCalled();
+ });
+
it("logs a failing source instead of throwing into the IPC listener", async () => {
const { session, engine, deliver } = createHarness();
engine.createSegmenter.mockImplementationOnce(() => {
diff --git a/apps/desktop/src/services/transcription/session.ts b/apps/desktop/src/services/transcription/session.ts
index 972fc98fd2..e63b098148 100644
--- a/apps/desktop/src/services/transcription/session.ts
+++ b/apps/desktop/src/services/transcription/session.ts
@@ -1,19 +1,17 @@
+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";
import { getLog } from "@triliumnext/core";
-import type { AudioCapture, AudioSource } from "./capture";
+import type { AudioCapture } from "./capture";
import { SAMPLE_RATE, type SpeechEngine, type SpeechSegment, type SpeechSegmenter } from "./engine";
-/** One transcribed stretch of speech. */
-export interface Utterance {
- source: AudioSource;
- /** Milliseconds from the start of the recording to the start of the utterance. */
- offsetMs: number;
- text: string;
-}
-
/** Receives the utterances of a recording in the order they were spoken per source. */
export interface TranscriptSink {
- write(utterance: Utterance): void | Promise<void>;
+ /**
+ * Called once before capturing starts, with the start time the offsets count from. A sink that
+ * throws here keeps the recording from starting.
+ */
+ open?(startedAt: number): void | Promise<void>;
+ write(utterance: TranscriptUtterance): void | Promise<void>;
/** Called once after the last utterance. */
close?(): void | Promise<void>;
}
@@ -31,7 +29,7 @@ export interface RecordingSessionOptions {
* to every sink.
*/
export class RecordingSession {
- private readonly pipelines = new Map<AudioSource, SourcePipeline>();
+ private readonly pipelines = new Map<TranscriptSource, SourcePipeline>();
private queue = Promise.resolve();
private startedAt = 0;
@@ -39,6 +37,9 @@ export class RecordingSession {
async start() {
this.startedAt = this.now();
+ for (const sink of this.options.sinks) {
+ await sink.open?.(this.startedAt);
+ }
return await this.options.capture.start((source, samples) => {
// An exception thrown from an `ipcMain` listener takes down the main process.
try {
@@ -67,7 +68,7 @@ export class RecordingSession {
}
/** Creates a source's pipeline on its first chunk, which was captured before it arrived. */
- private pipelineFor(source: AudioSource, firstChunk: Float32Array) {
+ private pipelineFor(source: TranscriptSource, firstChunk: Float32Array) {
let pipeline = this.pipelines.get(source);
if (!pipeline) {
const chunkMs = firstChunk.length / SAMPLE_RATE * 1000;
@@ -82,7 +83,7 @@ export class RecordingSession {
return pipeline;
}
- private enqueue(source: AudioSource, offsetMs: number, samples: Float32Array) {
+ private enqueue(source: TranscriptSource, offsetMs: number, samples: Float32Array) {
this.queue = this.queue.then(async () => {
const text = await this.options.engine.recognizer.transcribe(samples);
if (!text) {
@@ -104,32 +105,13 @@ export class RecordingSession {
}
}
-/** Logs every utterance, labeled "Me" for the microphone and "Others" for the system audio. */
-export class LogSink implements TranscriptSink {
- constructor(private readonly noteId: string) {}
-
- write({ source, offsetMs, text }: Utterance) {
- const speaker = source === "microphone" ? "Me" : "Others";
- getLog().info(`Transcription [${formatOffset(offsetMs)}] ${speaker}: ${text}`);
- }
-
- close() {
- getLog().info(`Transcription: recording for note ${this.noteId} stopped.`);
- }
-}
-
-export function formatOffset(milliseconds: number) {
- const seconds = Math.max(0, Math.floor(milliseconds / 1000));
- return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
-}
-
/** Segments one source and logs its peak level every 5 s, to spot a silent source. */
class SourcePipeline {
private peak = 0;
private peakSampleCount = 0;
constructor(
- private readonly source: AudioSource,
+ private readonly source: TranscriptSource,
private readonly segmenter: SpeechSegmenter,
/** Milliseconds from the start of the recording to the source's first sample. */
private readonly firstSampleOffsetMs: number,
diff --git a/apps/server/src/assets/translations/en/server.json b/apps/server/src/assets/translations/en/server.json
index 250e4a125c..bd9b62a74d 100644
--- a/apps/server/src/assets/translations/en/server.json
+++ b/apps/server/src/assets/translations/en/server.json
@@ -566,6 +566,7 @@
"already-recording": "A meeting is already being recorded.",
"note-not-found": "The meeting note no longer exists.",
"model-missing": "No speech recognition model was found in {{- directory}}.",
- "start-failed": "The recording could not start: {{- message}}"
+ "start-failed": "The recording could not start: {{- message}}",
+ "attachment-title": "Transcript"
}
}
diff --git a/packages/commons/src/index.ts b/packages/commons/src/index.ts
index 38bbfde4cb..6a22860188 100644
--- a/packages/commons/src/index.ts
+++ b/packages/commons/src/index.ts
@@ -56,4 +56,5 @@ export * from "./lib/marked_extensions.js";
// is imported by virtually every client module. Import it via its subpath instead,
// e.g. "@triliumnext/commons/src/lib/markdown_renderer".
export * from "./lib/task_states.js";
+export * from "./lib/transcript.js";
export * from "./lib/keyboard_shortcut_display.js";
diff --git a/packages/commons/src/lib/attachment_roles.ts b/packages/commons/src/lib/attachment_roles.ts
index 85c36a614d..b2b86f5fb7 100644
--- a/packages/commons/src/lib/attachment_roles.ts
+++ b/packages/commons/src/lib/attachment_roles.ts
@@ -96,7 +96,9 @@ export const ATTACHMENT_ROLES = {
/** Shapes saved into an Excalidraw canvas's library. */
canvasLibraryItem: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-shape-square" },
/** The file an import was read from, kept so an import that went wrong can be looked at again. */
- importSource: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-import" }
+ importSource: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-import" },
+ /** A meeting recording's transcript, which the recording rewrites as it transcribes. */
+ transcript: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-microphone" }
} as const satisfies Record<string, AttachmentRoleTraits>;
/** A role the app itself creates. Arbitrary strings reach the same fields — see {@link ATTACHMENT_ROLES}. */
diff --git a/packages/commons/src/lib/transcript.ts b/packages/commons/src/lib/transcript.ts
new file mode 100644
index 0000000000..541f9e319f
--- /dev/null
+++ b/packages/commons/src/lib/transcript.ts
@@ -0,0 +1,26 @@
+/**
+ * Where an utterance was heard: the local speaker's microphone, or everyone else on the system
+ * output.
+ */
+export type TranscriptSource = "microphone" | "system";
+
+/** One transcribed stretch of speech. */
+export interface TranscriptUtterance {
+ source: TranscriptSource;
+ /** Milliseconds from the start of the recording to the start of the utterance. */
+ offsetMs: number;
+ text: string;
+}
+
+/**
+ * The content of a `transcript` attachment: a meeting recording as it is transcribed. The recording
+ * rewrites it after every utterance.
+ */
+export interface Transcript {
+ version: 1;
+ /** When the recording started, as an ISO 8601 timestamp; the offsets count from here. */
+ startedAt: string;
+ utterances: TranscriptUtterance[];
+}
+
+export const TRANSCRIPT_MIME = "application/json";