Skip to content

Build your personal knowledge base with Trilium Notes

feat(attachments): show who is speaking before their words are transcribed

Elian Doranf2b550a7f203
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json
index 9912cd5894..1b2c20dabd 100644
--- a/apps/client/src/translations/en/translation.json
+++ b/apps/client/src/translations/en/translation.json
@@ -4589,6 +4589,7 @@
"speaker_me": "Me",
"speaker_others": "Others",
"empty": "Nothing has been transcribed yet.",
- "unreadable": "This transcript cannot be read."
+ "unreadable": "This transcript cannot be read.",
+ "speaking": "Speaking…"
}
}
diff --git a/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx b/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx
index 041f8fec5e..90dfa39808 100644
--- a/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx
+++ b/apps/client/src/widgets/launch_bar/RecordMeetingButton.tsx
@@ -56,7 +56,9 @@ function getAppearance(status: TranscriptionStatus, idleIcon: string, idleTitle:
/** Follows the main process's recording, which every window shares. */
export function useTranscriptionState() {
- const [ state, setState ] = useState<TranscriptionState>({ status: "idle", noteId: null });
+ const [ state, setState ] = useState<TranscriptionState>({
+ status: "idle", noteId: null, pendingSources: []
+ });
useEffect(() => {
const api = window.electronApi?.transcription;
diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.css b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.css
index 32f6a378b6..4fa52ec7ea 100644
--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.css
+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.css
@@ -42,4 +42,29 @@
p + p {
margin-top: 0.5rem;
}
+
+ .transcript-speaking {
+ display: flex;
+ align-items: center;
+ gap: 0.35rem;
+ font-style: italic;
+ color: var(--muted-text-color);
+
+ .bx {
+ font-size: 1.1em;
+ animation: transcript-speaking-pulse 1.2s ease-in-out infinite;
+ }
+ }
+}
+
+@keyframes transcript-speaking-pulse {
+ 50% {
+ opacity: 0.4;
+ }
+}
+
+@media (prefers-reduced-motion: reduce) {
+ .transcript .transcript-speaking .bx {
+ animation: none;
+ }
}
diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx
index b949fc4776..80692013c6 100644
--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx
+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx
@@ -1,4 +1,6 @@
-import type { Transcript, TranscriptUtterance } from "@triliumnext/commons";
+import type {
+ Transcript, TranscriptionState, TranscriptSource, TranscriptUtterance
+} from "@triliumnext/commons";
import { act } from "preact/test-utils";
import { describe, expect, it, vi } from "vitest";
@@ -93,6 +95,56 @@ describe("TranscriptView", () => {
expect(container.querySelectorAll(".transcript-microphone p")).toHaveLength(2);
});
+ it("shows who is speaking in the meeting its note records, before the words arrive", async () => {
+ let onStateChanged: (state: TranscriptionState) => void = () => {};
+ const recording = (noteId: string, pendingSources: TranscriptSource[]) =>
+ act(() => onStateChanged({ status: "recording", noteId, pendingSources }));
+ window.electronApi = {
+ transcription: {
+ getState: async () => ({
+ status: "recording", noteId: "meeting", pendingSources: []
+ }),
+ onStateChanged: (callback: typeof onStateChanged) => {
+ onStateChanged = callback;
+ return () => {};
+ }
+ }
+ } as unknown as typeof window.electronApi;
+ const getBlob = vi.fn()
+ .mockResolvedValueOnce({ content: transcript([]) })
+ .mockResolvedValueOnce({ content: transcript([
+ { source: "microphone", offsetMs: 1500, durationMs: 1000, text: "Hello?" }
+ ]) });
+ const attachment = { attachmentId: "transcript1", ownerId: "meeting", getBlob };
+ const component = new Component();
+ const container = renderInto(
+ <ParentComponent.Provider value={component}>
+ <TranscriptView attachment={attachment as unknown as FAttachment} />
+ </ParentComponent.Provider>
+ );
+ const turns = () => [ ...container.querySelectorAll(".transcript-turn") ]
+ .map((turn) => turn.textContent);
+ await vi.waitFor(() => expect(container.textContent).toContain("transcript.empty"));
+
+ await recording("meeting", [ "system" ]);
+ expect(turns()).toEqual([ "transcript.speaker_otherstranscript.speaking" ]);
+
+ const loadResults = new LoadResults([]);
+ loadResults.addAttachmentRow({ attachmentId: "transcript1", ownerId: "meeting" } as never);
+ await act(async () => {
+ await component.handleEvent("entitiesReloaded", { loadResults });
+ });
+ await recording("meeting", [ "microphone", "system" ]);
+ await vi.waitFor(() => expect(turns()).toEqual([
+ "transcript.speaker_me0:01Hello?transcript.speaking",
+ "transcript.speaker_otherstranscript.speaking"
+ ]));
+
+ await recording("other", [ "system" ]);
+ expect(turns()).toEqual([ "transcript.speaker_me0:01Hello?" ]);
+ window.electronApi = undefined;
+ });
+
it("follows new turns for a reader at the end, not for one who scrolled up", async () => {
const utterance = (index: number): TranscriptUtterance =>
({ source: "microphone", offsetMs: index * 60_000, text: `Line ${index}` });
diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx
index 0488da7c16..13f376f811 100644
--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx
+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx
@@ -5,6 +5,7 @@ import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "preac
import type FAttachment from "../../../entities/fattachment";
import { t } from "../../../services/i18n";
+import { useTranscriptionState } from "../../launch_bar/RecordMeetingButton";
import { useTriliumEvent } from "../../react/hooks";
import NoItems from "../../react/NoItems";
@@ -16,19 +17,22 @@ const PARAGRAPH_PAUSE_MS = 2000;
/**
* Shows a `transcript` attachment as the conversation it records, one paragraph per speaker turn.
- * Follows the attachment as a running recording rewrites it, and keeps a reader who is at the end
- * there as new turns arrive.
+ * Follows the attachment as a running recording rewrites it, shows which speakers the recording
+ * hears before their words are transcribed, and keeps a reader who is at the end there as new
+ * turns arrive.
*/
export default function TranscriptView({ attachment }: { attachment: FAttachment }) {
const [ content, setContent ] = useState<string | null>();
+ const recording = useTranscriptionState();
+ const pending = recording.noteId === attachment.ownerId ? recording.pendingSources : [];
const scrollerRef = useRef<HTMLDivElement>(null);
- const followRef = useRef(false);
+ /** The scroll height at the previous render, or `null` before the transcript first shows. */
+ const scrollHeightRef = useRef<number | null>(null);
const load = useCallback(() => {
let current = true;
void attachment.getBlob().then((blob) => {
if (current) {
- followRef.current = isScrolledToEnd(scrollerRef.current);
setContent(blob?.content ?? null);
}
});
@@ -39,10 +43,15 @@ export default function TranscriptView({ attachment }: { attachment: FAttachment
useEffect(load, [ load ]);
useLayoutEffect(() => {
const scroller = scrollerRef.current;
- if (followRef.current && scroller) {
+ if (!scroller) {
+ return;
+ }
+ const previousHeight = scrollHeightRef.current;
+ if (previousHeight !== null && wasAtEnd(scroller, previousHeight)) {
scroller.scrollTop = scroller.scrollHeight;
}
- }, [ content ]);
+ scrollHeightRef.current = scroller.scrollHeight;
+ });
useTriliumEvent("entitiesReloaded", ({ loadResults }) => {
const { attachmentId } = attachment;
if (loadResults.getAttachmentRows().some((row) => row.attachmentId === attachmentId)) {
@@ -58,19 +67,36 @@ export default function TranscriptView({ attachment }: { attachment: FAttachment
if (!transcript) {
return <NoItems icon="bx bx-error" text={t("transcript.unreadable")} />;
}
- if (!transcript.utterances.length) {
+ if (!transcript.utterances.length && !pending.length) {
return <NoItems icon="bx bx-microphone" text={t("transcript.empty")} />;
}
+ const turns = groupIntoTurns(transcript.utterances);
+ const lastTurn = turns.at(-1);
+ const speaking = (
+ <p className="transcript-speaking">
+ <span className="bx bx-user-voice" aria-hidden="true" />
+ {t("transcript.speaking")}
+ </p>
+ );
return (
<div ref={scrollerRef} className="transcript">
- {groupIntoTurns(transcript.utterances).map((turn) => (
+ {turns.map((turn) => (
<div key={turn.offsetMs} className={`transcript-turn transcript-${turn.source}`}>
<div className="transcript-turn-header">
<span className="transcript-speaker">{speakerName(turn.source)}</span>
<span className="transcript-time">{formatOffset(turn.offsetMs)}</span>
</div>
{turn.paragraphs.map((paragraph, index) => <p key={index}>{paragraph}</p>)}
+ {turn === lastTurn && pending.includes(turn.source) && speaking}
+ </div>
+ ))}
+ {pending.filter((source) => source !== lastTurn?.source).map((source) => (
+ <div key={`pending-${source}`} className={`transcript-turn transcript-${source}`}>
+ <div className="transcript-turn-header">
+ <span className="transcript-speaker">{speakerName(source)}</span>
+ </div>
+ {speaking}
</div>
))}
</div>
@@ -132,11 +158,9 @@ export function formatOffset(milliseconds: number) {
: `${minutes}:${seconds}`;
}
-function isScrolledToEnd(scroller: HTMLElement | null) {
- if (!scroller) {
- return false;
- }
- return scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight <= FOLLOW_THRESHOLD;
+/** Whether `scroller` showed the end of its content when that content was `scrollHeight` tall. */
+function wasAtEnd(scroller: HTMLElement, scrollHeight: number) {
+ return scrollHeight - scroller.scrollTop - scroller.clientHeight <= FOLLOW_THRESHOLD;
}
function speakerName(source: TranscriptSource) {
diff --git a/apps/desktop/src/services/transcription/engine.ts b/apps/desktop/src/services/transcription/engine.ts
index a69ea32ce3..98bf92709f 100644
--- a/apps/desktop/src/services/transcription/engine.ts
+++ b/apps/desktop/src/services/transcription/engine.ts
@@ -23,6 +23,8 @@ export interface SpeechSegmenter {
accept(samples: Float32Array): SpeechSegment[];
/** Ends the source and returns the utterance still in progress, if any. */
flush(): SpeechSegment[];
+ /** Whether the samples accepted so far end inside an utterance. */
+ isSpeaking(): boolean;
}
export interface SpeechEngine {
@@ -110,6 +112,10 @@ class VadSegmenter implements SpeechSegmenter {
return this.drain();
}
+ isSpeaking() {
+ return this.vad.isDetected();
+ }
+
private drain() {
const segments: SpeechSegment[] = [];
while (!this.vad.isEmpty()) {
diff --git a/apps/desktop/src/services/transcription/index.spec.ts b/apps/desktop/src/services/transcription/index.spec.ts
index 7b37bd06d7..e46931544f 100644
--- a/apps/desktop/src/services/transcription/index.spec.ts
+++ b/apps/desktop/src/services/transcription/index.spec.ts
@@ -1,11 +1,14 @@
import type { TranscriptionState } from "@triliumnext/commons";
import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { RecordingSessionOptions } from "./session";
+
const h = vi.hoisted(() => ({
handlers: new Map<string, (...args: unknown[]) => unknown>(),
broadcast: [] as TranscriptionState[],
notes: new Set<string>([ "meeting" ]),
loadEngine: vi.fn(),
+ sessionOptions: null as RecordingSessionOptions | null,
sessionStart: vi.fn(),
sessionStop: vi.fn()
}));
@@ -47,11 +50,17 @@ vi.mock("./session", () => ({
RecordingSession: class {
start = h.sessionStart;
stop = h.sessionStop;
+
+ constructor(options: RecordingSessionOptions) {
+ h.sessionOptions = options;
+ }
}
}));
const { ModelMissingError } = await import("./engine");
+const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };
+
async function loadService() {
vi.resetModules();
const service = await import("./index");
@@ -75,7 +84,7 @@ describe("transcription IPC", () => {
expect(await invoke("transcription-start", "meeting"))
.toBe("transcription.already-recording");
expect(await invoke("transcription-get-state"))
- .toEqual({ status: "recording", noteId: "meeting" });
+ .toEqual({ status: "recording", noteId: "meeting", pendingSources: [] });
await invoke("transcription-stop");
expect(h.sessionStop).toHaveBeenCalledOnce();
@@ -83,7 +92,23 @@ describe("transcription IPC", () => {
"starting", "starting", "recording", "recording",
"stopping", "stopping", "idle", "idle"
]);
- expect(await invoke("transcription-get-state")).toEqual({ status: "idle", noteId: null });
+ expect(await invoke("transcription-get-state")).toEqual(IDLE);
+ });
+
+ it("tells every window which sources are pending while it records", async () => {
+ const invoke = await loadService();
+ await invoke("transcription-start", "meeting");
+
+ h.sessionOptions?.onPendingChanged?.([ "system" ]);
+ expect(h.broadcast.at(-1)).toEqual({
+ status: "recording", noteId: "meeting", pendingSources: [ "system" ]
+ });
+ expect(await invoke("transcription-get-state")).toEqual(h.broadcast.at(-1));
+
+ await invoke("transcription-stop");
+ const stopping = h.broadcast.find((state) => state.status === "stopping");
+ expect(stopping?.pendingSources).toEqual([ "system" ]);
+ expect(h.broadcast.at(-1)).toEqual(IDLE);
});
it("refuses a note that does not exist, without changing state", async () => {
@@ -107,7 +132,7 @@ describe("transcription IPC", () => {
expect(await invoke("transcription-start", "meeting"))
.toBe(`transcription.start-failed {"message":"No audio source could be captured."}`);
- expect(await invoke("transcription-get-state")).toEqual({ status: "idle", noteId: null });
+ expect(await invoke("transcription-get-state")).toEqual(IDLE);
await invoke("transcription-stop");
expect(h.sessionStop).not.toHaveBeenCalled();
});
diff --git a/apps/desktop/src/services/transcription/index.ts b/apps/desktop/src/services/transcription/index.ts
index 45fa601dfb..a2432b54b0 100644
--- a/apps/desktop/src/services/transcription/index.ts
+++ b/apps/desktop/src/services/transcription/index.ts
@@ -8,7 +8,9 @@ import { ElectronAudioCapture } from "./capture";
import { loadSherpaEngine, ModelMissingError } from "./engine";
import { RecordingSession, type TranscriptSink } from "./session";
-let state: TranscriptionState = { status: "idle", noteId: null };
+const IDLE: TranscriptionState = { status: "idle", noteId: null, pendingSources: [] };
+
+let state = IDLE;
let activeSession: RecordingSession | null = null;
export function setupTranscriptionHandlers() {
@@ -30,20 +32,21 @@ export async function startRecording(noteId: unknown): Promise<string> {
return t("transcription.note-not-found");
}
- setState({ status: "starting", noteId });
+ setState({ status: "starting", noteId, pendingSources: [] });
try {
const session = new RecordingSession({
capture: new ElectronAudioCapture(),
engine: await loadSherpaEngine(),
- sinks: createSinks(noteId)
+ sinks: createSinks(noteId),
+ onPendingChanged: (pendingSources) => setState({ ...state, pendingSources })
});
const sources = await session.start();
activeSession = session;
getLog().info(`Transcription: recording ${sources.join(" and ")} for note ${noteId}.`);
- setState({ status: "recording", noteId });
+ setState({ ...state, status: "recording" });
return "";
} catch (e) {
- setState({ status: "idle", noteId: null });
+ setState(IDLE);
if (e instanceof ModelMissingError) {
return t("transcription.model-missing", { directory: e.directory });
}
@@ -58,13 +61,13 @@ export async function stopRecording() {
return;
}
- setState({ status: "stopping", noteId: state.noteId });
+ setState({ ...state, status: "stopping" });
activeSession = null;
try {
await session.stop();
getLog().info(`Transcription: recording for note ${state.noteId} stopped.`);
} finally {
- setState({ status: "idle", noteId: null });
+ setState(IDLE);
}
}
diff --git a/apps/desktop/src/services/transcription/session.spec.ts b/apps/desktop/src/services/transcription/session.spec.ts
index ed084847cd..34666fa002 100644
--- a/apps/desktop/src/services/transcription/session.spec.ts
+++ b/apps/desktop/src/services/transcription/session.spec.ts
@@ -22,6 +22,16 @@ function utteranceChunk(id: number, start: number) {
return samples;
}
+/** The `id` of a chunk that completes no utterance. */
+const NO_SEGMENT = -1;
+
+/** A chunk that completes no utterance and leaves the fake segmenter speaking or not. */
+function speechChunk(speaking: boolean) {
+ const samples = utteranceChunk(NO_SEGMENT, 0);
+ samples[2] = speaking ? 1 : 0;
+ return samples;
+}
+
function createHarness(extraSinks: TranscriptSink[] = []) {
let clock = 1000;
let onSamples: SamplesListener = () => {};
@@ -40,9 +50,17 @@ function createHarness(extraSinks: TranscriptSink[] = []) {
},
createSegmenter: vi.fn(() => {
const pending = flushed.shift() ?? [];
+ let speaking = false;
return {
- accept: (samples: Float32Array) => [ { start: samples[1], samples } ],
- flush: () => pending
+ accept: (samples: Float32Array) => {
+ speaking = samples[2] === 1;
+ return samples[0] === NO_SEGMENT ? [] : [ { start: samples[1], samples } ];
+ },
+ flush: () => {
+ speaking = false;
+ return pending;
+ },
+ isSpeaking: () => speaking
};
})
};
@@ -55,10 +73,13 @@ function createHarness(extraSinks: TranscriptSink[] = []) {
close: vi.fn()
};
const sinks = [ ...extraSinks, sink ];
- const session = new RecordingSession({ capture, engine, sinks, now: () => clock });
+ const onPendingChanged = vi.fn();
+ const session = new RecordingSession({
+ capture, engine, sinks, onPendingChanged, now: () => clock
+ });
return {
- session, capture, engine, sink, written, flushed,
+ session, capture, engine, sink, written, flushed, onPendingChanged,
/** Delivers `samples` from `source` when the clock reads `at`. */
deliver(source: TranscriptSource, samples: Float32Array, at: number) {
clock = at;
@@ -118,6 +139,35 @@ describe("RecordingSession", () => {
expect(log.error).toHaveBeenCalledTimes(2);
});
+ it("reports a source as pending from its first speech until its utterance is written", async () => {
+ const { session, engine, sink, deliver, onPendingChanged } = createHarness();
+ let finishTranscribing = () => {};
+ engine.recognizer.transcribe.mockImplementationOnce(() => new Promise((resolve) => {
+ finishTranscribing = () => resolve("utterance 1");
+ }));
+ await session.start();
+
+ deliver("microphone", speechChunk(true), 2000);
+ deliver("system", speechChunk(true), 2000);
+ deliver("microphone", speechChunk(true), 3000);
+ // The utterance ends: the microphone stays pending while it is transcribed.
+ deliver("microphone", utteranceChunk(1, 0), 4000);
+ deliver("system", speechChunk(false), 4000);
+ expect(onPendingChanged.mock.calls).toEqual([
+ [ [ "microphone" ] ],
+ [ [ "microphone", "system" ] ],
+ [ [ "microphone" ] ]
+ ]);
+
+ await vi.waitFor(() => expect(engine.recognizer.transcribe).toHaveBeenCalledOnce());
+ finishTranscribing();
+ await vi.waitFor(() => expect(onPendingChanged).toHaveBeenLastCalledWith([]));
+ expect(sink.write.mock.invocationCallOrder[0])
+ .toBeLessThan(onPendingChanged.mock.invocationCallOrder[3]);
+ await session.stop();
+ expect(onPendingChanged).toHaveBeenCalledTimes(4);
+ });
+
it("does not start capturing when a sink cannot open", async () => {
const { session, capture, sink } = createHarness();
sink.open.mockImplementationOnce(() => {
diff --git a/apps/desktop/src/services/transcription/session.ts b/apps/desktop/src/services/transcription/session.ts
index 4da6ea0409..a6e7dacea6 100644
--- a/apps/desktop/src/services/transcription/session.ts
+++ b/apps/desktop/src/services/transcription/session.ts
@@ -20,6 +20,11 @@ export interface RecordingSessionOptions {
capture: AudioCapture;
engine: SpeechEngine;
sinks: TranscriptSink[];
+ /**
+ * Called with the sources whose speech has not reached the sinks yet, either because it is
+ * still being spoken or because it waits to be transcribed, every time that list changes.
+ */
+ onPendingChanged?: (sources: TranscriptSource[]) => void;
now?: () => number;
}
@@ -30,8 +35,11 @@ export interface RecordingSessionOptions {
*/
export class RecordingSession {
private readonly pipelines = new Map<TranscriptSource, SourcePipeline>();
+ /** How many utterances of each source wait to be transcribed or written. */
+ private readonly queued = new Map<TranscriptSource, number>();
private queue = Promise.resolve();
private startedAt = 0;
+ private pendingSources: TranscriptSource[] = [];
constructor(private readonly options: RecordingSessionOptions) {}
@@ -44,6 +52,7 @@ export class RecordingSession {
// An exception thrown from an `ipcMain` listener takes down the main process.
try {
this.pipelineFor(source, samples).accept(samples);
+ this.updatePending();
} catch (e) {
getLog().error(`Transcription: ${source} failed: ${(e as Error)?.message ?? e}`);
}
@@ -56,6 +65,7 @@ export class RecordingSession {
for (const pipeline of this.pipelines.values()) {
pipeline.flush();
}
+ this.updatePending();
await this.queue;
for (const sink of this.options.sinks) {
@@ -84,6 +94,7 @@ export class RecordingSession {
}
private enqueue(source: TranscriptSource, offsetMs: number, samples: Float32Array) {
+ this.queued.set(source, (this.queued.get(source) ?? 0) + 1);
this.queue = this.queue.then(async () => {
const text = await this.options.engine.recognizer.transcribe(samples);
if (!text) {
@@ -98,7 +109,22 @@ export class RecordingSession {
getLog().error(`Transcription: a sink failed: ${e}`);
}
}
- }).catch((e) => getLog().error(`Transcription failed: ${e}`));
+ }).catch((e) => getLog().error(`Transcription failed: ${e}`)).finally(() => {
+ this.queued.set(source, (this.queued.get(source) ?? 1) - 1);
+ this.updatePending();
+ });
+ }
+
+ /** Reports the sources with speech not yet written, when they differ from the last report. */
+ private updatePending() {
+ const pending = [ ...this.pipelines ]
+ .filter(([ source, pipeline ]) => pipeline.isSpeaking() || this.queued.get(source))
+ .map(([ source ]) => source)
+ .sort();
+ if (pending.join() !== this.pendingSources.join()) {
+ this.pendingSources = pending;
+ this.options.onPendingChanged?.(pending);
+ }
}
private now() {
@@ -128,6 +154,10 @@ class SourcePipeline {
this.emit(this.segmenter.flush());
}
+ isSpeaking() {
+ return this.segmenter.isSpeaking();
+ }
+
private emit(segments: SpeechSegment[]) {
for (const segment of segments) {
const offsetMs = this.firstSampleOffsetMs + segment.start / SAMPLE_RATE * 1000;
diff --git a/apps/desktop/src/sherpa-onnx-node.d.ts b/apps/desktop/src/sherpa-onnx-node.d.ts
index 5f3ad0d8da..5f9ce1def2 100644
--- a/apps/desktop/src/sherpa-onnx-node.d.ts
+++ b/apps/desktop/src/sherpa-onnx-node.d.ts
@@ -13,6 +13,7 @@ declare module "sherpa-onnx-node" {
constructor(config: object, bufferSizeInSeconds: number);
acceptWaveform(samples: Float32Array): void;
isEmpty(): boolean;
+ isDetected(): boolean;
front(enableExternalBuffer: boolean): { start: number; samples: Float32Array };
pop(): void;
flush(): void;
diff --git a/packages/commons/src/lib/electron_api_interface.ts b/packages/commons/src/lib/electron_api_interface.ts
index 5e44dce8e1..80e6fad3af 100644
--- a/packages/commons/src/lib/electron_api_interface.ts
+++ b/packages/commons/src/lib/electron_api_interface.ts
@@ -1,4 +1,5 @@
import type { SecurityToggleApi } from "./security_settings.js";
+import type { TranscriptSource } from "./transcript.js";
/**
* Parameters delivered by the main process to the renderer when the user opens
@@ -721,6 +722,8 @@ export interface TranscriptionState {
status: TranscriptionStatus;
/** The meeting note the recording belongs to, or `null` when idle. */
noteId: string | null;
+ /** The sources heard speaking, or whose speech is still being transcribed. */
+ pendingSources: TranscriptSource[];
}
/**