Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(desktop): stream the meeting transcript into an attachment
Elian Doran4c2efa1c280a
diff --git a/apps/client/src/services/attachment_groups.ts b/apps/client/src/services/attachment_groups.tsindex 2e04f08896..87ceda32ca 100644--- a/apps/client/src/services/attachment_groups.ts+++ b/apps/client/src/services/attachment_groups.ts@@ -35,7 +35,9 @@ export const ATTACHMENT_ROLE_GROUPS: Record<AttachmentRole, AttachmentGroup> = {// than attached to the note by hand.canvasLibraryItem: "system",// The file an import was read from, kept so an import that went wrong can be looked at again.- importSource: "system"+ importSource: "system",+ // A meeting recording's transcript: made by the app, but what the reader asked for.+ transcript: "user"};/**diff --git a/apps/client/src/services/attachment_role_names.spec.ts b/apps/client/src/services/attachment_role_names.spec.tsindex 6f36d3e2fd..8f21e545ad 100644--- a/apps/client/src/services/attachment_role_names.spec.ts+++ b/apps/client/src/services/attachment_role_names.spec.ts@@ -10,7 +10,7 @@ const { attachmentRoleLabel } = await import("./attachment_role_names.js");describe("attachmentRoleLabel", () => {it("names the app's own doing, which nothing else in the card does", () => {// "Site icon" is the whole reason that row is not a picture the reader forgot placing.- for (const role of [ "favicon", "coverImage", "viewConfig", "canvasLibraryItem", "importSource" ]) {+ for (const role of [ "favicon", "coverImage", "viewConfig", "canvasLibraryItem", "importSource", "transcript" ]) {expect(attachmentRoleLabel(role), role).toBe(`attachment_roles.${role}`);}});diff --git a/apps/client/src/services/attachment_role_names.ts b/apps/client/src/services/attachment_role_names.tsindex 41d0fada9d..7322473a07 100644--- a/apps/client/src/services/attachment_role_names.ts+++ b/apps/client/src/services/attachment_role_names.ts@@ -24,7 +24,8 @@ const ATTACHMENT_ROLE_NAMES: Record<NamedAttachmentRole, () => string> = {coverImage: () => t("attachment_roles.coverImage"),viewConfig: () => t("attachment_roles.viewConfig"),canvasLibraryItem: () => t("attachment_roles.canvasLibraryItem"),- importSource: () => t("attachment_roles.importSource")+ importSource: () => t("attachment_roles.importSource"),+ transcript: () => t("attachment_roles.transcript")};/**diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.jsonindex af734a4189..c463114f7d 100644--- a/apps/client/src/translations/en/translation.json+++ b/apps/client/src/translations/en/translation.json@@ -2443,7 +2443,8 @@"coverImage": "Cover image","viewConfig": "View settings","canvasLibraryItem": "Canvas library",- "importSource": "Import source"+ "importSource": "Import source",+ "transcript": "Meeting transcript"},"bookmark_switch": {"bookmark": "Bookmark"diff --git a/apps/desktop/src/services/transcription/attachment_sink.spec.ts b/apps/desktop/src/services/transcription/attachment_sink.spec.tsnew file mode 100644index 0000000000..141e7c9c18--- /dev/null+++ b/apps/desktop/src/services/transcription/attachment_sink.spec.ts@@ -0,0 +1,72 @@+import type { Transcript } from "@triliumnext/commons";+import { beforeEach, describe, expect, it, vi } from "vitest";++const h = vi.hoisted(() => ({+ saveAttachment: vi.fn(),+ setContent: vi.fn(),+ transactions: 0+}));++vi.mock("@triliumnext/core", () => ({+ becca: {+ getNoteOrThrow: (noteId: string) => ({ noteId, saveAttachment: h.saveAttachment }),+ getAttachmentOrThrow: (attachmentId: string) => ({ attachmentId, setContent: h.setContent })+ },+ cls: { init: <T>(callback: () => T) => callback() },+ getSql: () => ({+ transactional: <T>(callback: () => T) => {+ h.transactions++;+ return callback();+ }+ })+}));+vi.mock("i18next", () => ({ t: (key: string) => key }));++const { AttachmentSink } = await import("./attachment_sink");++const STARTED_AT = Date.UTC(2026, 9, 13, 14, 30);++beforeEach(() => {+ vi.clearAllMocks();+ h.transactions = 0;+ h.saveAttachment.mockReturnValue({ attachmentId: "transcript1" });+});++describe("AttachmentSink", () => {+ it("creates an empty transcript attachment on the meeting note when it opens", () => {+ new AttachmentSink("meeting").open(STARTED_AT);++ expect(h.saveAttachment).toHaveBeenCalledExactlyOnceWith({+ role: "transcript",+ mime: "application/json",+ title: "transcription.attachment-title",+ content: expect.any(String)+ });+ const content = JSON.parse(h.saveAttachment.mock.calls[0][0].content) as Transcript;+ expect(content).toEqual({+ version: 1, startedAt: "2026-10-13T14:30:00.000Z", utterances: []+ });+ });++ it("rewrites the whole transcript after every utterance, in a transaction each", () => {+ const sink = new AttachmentSink("meeting");+ sink.open(STARTED_AT);+ sink.write({ source: "microphone", offsetMs: 1500, text: "Can you hear me?" });+ sink.write({ source: "system", offsetMs: 3000, text: "Yes." });++ expect(h.setContent).toHaveBeenCalledTimes(2);+ const last = JSON.parse(h.setContent.mock.calls[1][0]) as Transcript;+ expect(last.utterances).toEqual([+ { source: "microphone", offsetMs: 1500, text: "Can you hear me?" },+ { source: "system", offsetMs: 3000, text: "Yes." }+ ]);+ expect(h.transactions).toBe(3);+ });++ it("refuses to write before it is open", () => {+ const sink = new AttachmentSink("meeting");+ expect(() => sink.write({ source: "system", offsetMs: 0, text: "Hi" }))+ .toThrow("The transcript attachment was not created.");+ expect(h.setContent).not.toHaveBeenCalled();+ });+});diff --git a/apps/desktop/src/services/transcription/attachment_sink.ts b/apps/desktop/src/services/transcription/attachment_sink.tsnew file mode 100644index 0000000000..98ef507ac6--- /dev/null+++ b/apps/desktop/src/services/transcription/attachment_sink.ts@@ -0,0 +1,50 @@+import { type Transcript, TRANSCRIPT_MIME, type TranscriptUtterance } from "@triliumnext/commons";+import { becca, cls, getSql } from "@triliumnext/core";+import { t } from "i18next";++import type { TranscriptSink } from "./session";++/**+ * Keeps the transcript in a `transcript` attachment of the meeting note, rewritten after every+ * utterance. Each write commits its own transaction, which sends the change to every open window.+ */+export class AttachmentSink implements TranscriptSink {+ private transcript: Transcript | null = null;+ private attachmentId: string | null = null;++ constructor(private readonly noteId: string) {}++ open(startedAt: number) {+ this.transcript = {+ version: 1,+ startedAt: new Date(startedAt).toISOString(),+ utterances: []+ };+ const content = serialize(this.transcript);+ this.attachmentId = inTransaction(() => becca.getNoteOrThrow(this.noteId).saveAttachment({+ role: "transcript",+ mime: TRANSCRIPT_MIME,+ title: t("transcription.attachment-title"),+ content+ }).attachmentId) ?? null;+ }++ write(utterance: TranscriptUtterance) {+ const { transcript, attachmentId } = this;+ if (!transcript || !attachmentId) {+ throw new Error("The transcript attachment was not created.");+ }++ transcript.utterances.push(utterance);+ const content = serialize(transcript);+ inTransaction(() => becca.getAttachmentOrThrow(attachmentId).setContent(content));+ }+}++function serialize(transcript: Transcript) {+ return JSON.stringify(transcript, null, 2);+}++function inTransaction<T>(callback: () => T): T {+ return cls.init(() => getSql().transactional(callback));+}diff --git a/apps/desktop/src/services/transcription/capture.ts b/apps/desktop/src/services/transcription/capture.tsindex 0834e99124..fdb7b18b68 100644--- a/apps/desktop/src/services/transcription/capture.ts+++ b/apps/desktop/src/services/transcription/capture.ts@@ -1,3 +1,4 @@+import type { TranscriptSource } from "@triliumnext/commons";import { getLog } from "@triliumnext/core";import { app, BrowserWindow, ipcMain, type IpcMainEvent, session } from "electron";import fs from "fs";@@ -5,10 +6,7 @@ import path from "path";import { SAMPLE_RATE } from "./engine";-/** The microphone carries the local speaker; the system output carries everyone else. */-export type AudioSource = "microphone" | "system";--export type SamplesListener = (source: AudioSource, samples: Float32Array) => void;+export type SamplesListener = (source: TranscriptSource, samples: Float32Array) => void;/** Produces mono {@link SAMPLE_RATE} samples per source until stopped. */export interface AudioCapture {@@ -17,7 +15,7 @@ export interface AudioCapture {** @throws when no source could start.*/- start(onSamples: SamplesListener): Promise<AudioSource[]>;+ start(onSamples: SamplesListener): Promise<TranscriptSource[]>;stop(): void;}@@ -29,7 +27,8 @@ const SOURCE_STARTERS = [[ "system", "captureSystemAudio()" ]] as const;-type SamplesIpcListener = (event: IpcMainEvent, source: AudioSource, samples: Float32Array) => void;+type SamplesIpcListener =+ (event: IpcMainEvent, source: TranscriptSource, samples: Float32Array) => void;/*** Captures the default microphone and the system audio output in a hidden window, which streams@@ -50,7 +49,7 @@ export class ElectronAudioCapture implements AudioCapture {};ipcMain.on(SAMPLES_CHANNEL, this.listener);- const sources: AudioSource[] = [];+ const sources: TranscriptSource[] = [];for (const [ source, call ] of SOURCE_STARTERS) {try {// getDisplayMedia() requires a user gesture, which the hidden window only getsdiff --git a/apps/desktop/src/services/transcription/index.spec.ts b/apps/desktop/src/services/transcription/index.spec.tsindex 08e46b8a9c..7b37bd06d7 100644--- a/apps/desktop/src/services/transcription/index.spec.ts+++ b/apps/desktop/src/services/transcription/index.spec.ts@@ -42,8 +42,8 @@ vi.mock("./engine", async (importOriginal) => ({...(await importOriginal<typeof import("./engine")>()),loadSherpaEngine: h.loadEngine}));+vi.mock("./attachment_sink", () => ({ AttachmentSink: class {} }));vi.mock("./session", () => ({- LogSink: class {},RecordingSession: class {start = h.sessionStart;stop = h.sessionStop;diff --git a/apps/desktop/src/services/transcription/index.ts b/apps/desktop/src/services/transcription/index.tsindex f2fa140b6c..45fa601dfb 100644--- a/apps/desktop/src/services/transcription/index.ts+++ b/apps/desktop/src/services/transcription/index.ts@@ -3,9 +3,10 @@ import { becca, getLog } from "@triliumnext/core";import { BrowserWindow, ipcMain } from "electron";import { t } from "i18next";+import { AttachmentSink } from "./attachment_sink";import { ElectronAudioCapture } from "./capture";import { loadSherpaEngine, ModelMissingError } from "./engine";-import { LogSink, RecordingSession, type TranscriptSink } from "./session";+import { RecordingSession, type TranscriptSink } from "./session";let state: TranscriptionState = { status: "idle", noteId: null };let activeSession: RecordingSession | null = null;@@ -61,6 +62,7 @@ export async function stopRecording() {activeSession = null;try {await session.stop();+ getLog().info(`Transcription: recording for note ${state.noteId} stopped.`);} finally {setState({ status: "idle", noteId: null });}@@ -72,7 +74,7 @@ export function getState() {/** Where the utterances of a recording for `noteId` go. */function createSinks(noteId: string): TranscriptSink[] {- return [ new LogSink(noteId) ];+ return [ new AttachmentSink(noteId) ];}function setState(newState: TranscriptionState) {diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.tsindex c851e61ae9..9e32f57034 100644--- a/apps/desktop/src/services/transcription/session.spec.ts+++ b/apps/desktop/src/services/transcription/session.spec.ts@@ -1,8 +1,9 @@+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";import { beforeEach, describe, expect, it, vi } from "vitest";-import type { AudioSource, SamplesListener } from "./capture";+import type { SamplesListener } from "./capture";import type { SpeechSegment } from "./engine";-import { RecordingSession, type TranscriptSink, type Utterance } from "./session";+import { RecordingSession, type TranscriptSink } from "./session";const log = vi.hoisted(() => ({ info: vi.fn(), error: vi.fn() }));vi.mock("@triliumnext/core", () => ({ getLog: () => log }));@@ -27,7 +28,7 @@ function createHarness(extraSinks: TranscriptSink[] = []) {const capture = {start: vi.fn(async (listener: SamplesListener) => {onSamples = listener;- return [ "microphone", "system" ] as AudioSource[];+ return [ "microphone", "system" ] as TranscriptSource[];}),stop: vi.fn()};@@ -45,9 +46,10 @@ function createHarness(extraSinks: TranscriptSink[] = []) {};})};- const written: Utterance[] = [];+ const written: TranscriptUtterance[] = [];const sink = {- write: vi.fn((utterance: Utterance) => {+ open: vi.fn(),+ write: vi.fn((utterance: TranscriptUtterance) => {written.push(utterance);}),close: vi.fn()@@ -58,7 +60,7 @@ function createHarness(extraSinks: TranscriptSink[] = []) {return {session, capture, engine, sink, written, flushed,/** Delivers `samples` from `source` when the clock reads `at`. */- deliver(source: AudioSource, samples: Float32Array, at: number) {+ deliver(source: TranscriptSource, samples: Float32Array, at: number) {clock = at;onSamples(source, samples);}@@ -71,8 +73,11 @@ beforeEach(() => {describe("RecordingSession", () => {it("labels each source's utterances with their offset from the recording start", async () => {- const { session, engine, deliver, written } = createHarness();+ const { session, capture, engine, sink, deliver, written } = createHarness();await session.start();+ expect(sink.open).toHaveBeenCalledExactlyOnceWith(1000);+ expect(sink.open.mock.invocationCallOrder[0])+ .toBeLessThan(capture.start.mock.invocationCallOrder[0]);// The microphone's first one-second chunk arrives 2 s in, so it began 1 s in.deliver("microphone", utteranceChunk(1, 8000), 3000);@@ -113,6 +118,16 @@ describe("RecordingSession", () => {expect(log.error).toHaveBeenCalledTimes(2);});+ it("does not start capturing when a sink cannot open", async () => {+ const { session, capture, sink } = createHarness();+ sink.open.mockImplementationOnce(() => {+ throw new Error("note deleted");+ });++ await expect(session.start()).rejects.toThrow("note deleted");+ expect(capture.start).not.toHaveBeenCalled();+ });+it("logs a failing source instead of throwing into the IPC listener", async () => {const { session, engine, deliver } = createHarness();engine.createSegmenter.mockImplementationOnce(() => {diff --git a/apps/desktop/src/services/transcription/session.ts b/apps/desktop/src/services/transcription/session.tsindex 972fc98fd2..e63b098148 100644--- a/apps/desktop/src/services/transcription/session.ts+++ b/apps/desktop/src/services/transcription/session.ts@@ -1,19 +1,17 @@+import type { TranscriptSource, TranscriptUtterance } from "@triliumnext/commons";import { getLog } from "@triliumnext/core";-import type { AudioCapture, AudioSource } from "./capture";+import type { AudioCapture } from "./capture";import { SAMPLE_RATE, type SpeechEngine, type SpeechSegment, type SpeechSegmenter } from "./engine";-/** One transcribed stretch of speech. */-export interface Utterance {- source: AudioSource;- /** Milliseconds from the start of the recording to the start of the utterance. */- offsetMs: number;- text: string;-}-/** Receives the utterances of a recording in the order they were spoken per source. */export interface TranscriptSink {- write(utterance: Utterance): void | Promise<void>;+ /**+ * Called once before capturing starts, with the start time the offsets count from. A sink that+ * throws here keeps the recording from starting.+ */+ open?(startedAt: number): void | Promise<void>;+ write(utterance: TranscriptUtterance): void | Promise<void>;/** Called once after the last utterance. */close?(): void | Promise<void>;}@@ -31,7 +29,7 @@ export interface RecordingSessionOptions {* to every sink.*/export class RecordingSession {- private readonly pipelines = new Map<AudioSource, SourcePipeline>();+ private readonly pipelines = new Map<TranscriptSource, SourcePipeline>();private queue = Promise.resolve();private startedAt = 0;@@ -39,6 +37,9 @@ export class RecordingSession {async start() {this.startedAt = this.now();+ for (const sink of this.options.sinks) {+ await sink.open?.(this.startedAt);+ }return await this.options.capture.start((source, samples) => {// An exception thrown from an `ipcMain` listener takes down the main process.try {@@ -67,7 +68,7 @@ export class RecordingSession {}/** Creates a source's pipeline on its first chunk, which was captured before it arrived. */- private pipelineFor(source: AudioSource, firstChunk: Float32Array) {+ private pipelineFor(source: TranscriptSource, firstChunk: Float32Array) {let pipeline = this.pipelines.get(source);if (!pipeline) {const chunkMs = firstChunk.length / SAMPLE_RATE * 1000;@@ -82,7 +83,7 @@ export class RecordingSession {return pipeline;}- private enqueue(source: AudioSource, offsetMs: number, samples: Float32Array) {+ private enqueue(source: TranscriptSource, offsetMs: number, samples: Float32Array) {this.queue = this.queue.then(async () => {const text = await this.options.engine.recognizer.transcribe(samples);if (!text) {@@ -104,32 +105,13 @@ export class RecordingSession {}}-/** Logs every utterance, labeled "Me" for the microphone and "Others" for the system audio. */-export class LogSink implements TranscriptSink {- constructor(private readonly noteId: string) {}-- write({ source, offsetMs, text }: Utterance) {- const speaker = source === "microphone" ? "Me" : "Others";- getLog().info(`Transcription [${formatOffset(offsetMs)}] ${speaker}: ${text}`);- }-- close() {- getLog().info(`Transcription: recording for note ${this.noteId} stopped.`);- }-}--export function formatOffset(milliseconds: number) {- const seconds = Math.max(0, Math.floor(milliseconds / 1000));- return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;-}-/** Segments one source and logs its peak level every 5 s, to spot a silent source. */class SourcePipeline {private peak = 0;private peakSampleCount = 0;constructor(- private readonly source: AudioSource,+ private readonly source: TranscriptSource,private readonly segmenter: SpeechSegmenter,/** Milliseconds from the start of the recording to the source's first sample. */private readonly firstSampleOffsetMs: number,diff --git a/apps/server/src/assets/translations/en/server.json b/apps/server/src/assets/translations/en/server.jsonindex 250e4a125c..bd9b62a74d 100644--- a/apps/server/src/assets/translations/en/server.json+++ b/apps/server/src/assets/translations/en/server.json@@ -566,6 +566,7 @@"already-recording": "A meeting is already being recorded.","note-not-found": "The meeting note no longer exists.","model-missing": "No speech recognition model was found in {{- directory}}.",- "start-failed": "The recording could not start: {{- message}}"+ "start-failed": "The recording could not start: {{- message}}",+ "attachment-title": "Transcript"}}diff --git a/packages/commons/src/index.ts b/packages/commons/src/index.tsindex 38bbfde4cb..6a22860188 100644--- a/packages/commons/src/index.ts+++ b/packages/commons/src/index.ts@@ -56,4 +56,5 @@ export * from "./lib/marked_extensions.js";// is imported by virtually every client module. Import it via its subpath instead,// e.g. "@triliumnext/commons/src/lib/markdown_renderer".export * from "./lib/task_states.js";+export * from "./lib/transcript.js";export * from "./lib/keyboard_shortcut_display.js";diff --git a/packages/commons/src/lib/attachment_roles.ts b/packages/commons/src/lib/attachment_roles.tsindex 85c36a614d..b2b86f5fb7 100644--- a/packages/commons/src/lib/attachment_roles.ts+++ b/packages/commons/src/lib/attachment_roles.ts@@ -96,7 +96,9 @@ export const ATTACHMENT_ROLES = {/** Shapes saved into an Excalidraw canvas's library. */canvasLibraryItem: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-shape-square" },/** The file an import was read from, kept so an import that went wrong can be looked at again. */- importSource: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-import" }+ importSource: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-import" },+ /** A meeting recording's transcript, which the recording rewrites as it transcribes. */+ transcript: { picture: false, deduplicated: false, embedded: false, copiedAs: "file", icon: "bx bx-microphone" }} as const satisfies Record<string, AttachmentRoleTraits>;/** A role the app itself creates. Arbitrary strings reach the same fields — see {@link ATTACHMENT_ROLES}. */diff --git a/packages/commons/src/lib/transcript.ts b/packages/commons/src/lib/transcript.tsnew file mode 100644index 0000000000..541f9e319f--- /dev/null+++ b/packages/commons/src/lib/transcript.ts@@ -0,0 +1,26 @@+/**+ * Where an utterance was heard: the local speaker's microphone, or everyone else on the system+ * output.+ */+export type TranscriptSource = "microphone" | "system";++/** One transcribed stretch of speech. */+export interface TranscriptUtterance {+ source: TranscriptSource;+ /** Milliseconds from the start of the recording to the start of the utterance. */+ offsetMs: number;+ text: string;+}++/**+ * The content of a `transcript` attachment: a meeting recording as it is transcribed. The recording+ * rewrites it after every utterance.+ */+export interface Transcript {+ version: 1;+ /** When the recording started, as an ISO 8601 timestamp; the offsets count from here. */+ startedAt: string;+ utterances: TranscriptUtterance[];+}++export const TRANSCRIPT_MIME = "application/json";