Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(transcript): hide filler words and stutters
Elian Doran82b723631b5b
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.jsonindex c06c267564..bf6a145de0 100644--- a/apps/client/src/translations/en/translation.json+++ b/apps/client/src/translations/en/translation.json@@ -4593,6 +4593,8 @@"empty": "Nothing has been transcribed yet.","unreadable": "This transcript cannot be read.","speaking": "Speaking…",- "offset": "{{offset}} into the transcription"+ "offset": "{{offset}} into the transcription",+ "cleanup": "Hide filler words",+ "cleanup_tooltip": "Hides “uh”, “um” and stuttered words. The transcript itself is not changed."}}diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.css b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.cssindex f5d6385afe..4f83e8af6c 100644--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.css+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.css@@ -5,6 +5,15 @@padding: 0.5rem 1rem 1rem;user-select: text;+ .transcript-options {+ display: flex;+ justify-content: flex-end;+ max-width: var(--preferred-max-content-width, 900px);+ margin-bottom: 0.5rem;+ color: var(--muted-text-color);+ font-size: 0.9em;+ }+.transcript-turn {max-width: var(--preferred-max-content-width, 900px);margin-bottom: 1rem;diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsxindex 21a660c479..07daa4dd35 100644--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.spec.tsx@@ -8,16 +8,19 @@ import { describe, expect, it, vi } from "vitest";import Component from "../../../components/component";import type FAttachment from "../../../entities/fattachment";import LoadResults from "../../../services/load_results";+import options from "../../../services/options";import { renderInto } from "../../../test/render";import { ParentComponent } from "../../react/react_utils";import TranscriptView, {formatOffset, groupIntoTurns, type TranscriptTurn} from "./TranscriptView";-const h = vi.hoisted(() => ({ post: vi.fn() }));+const h = vi.hoisted(() => ({ post: vi.fn(), put: vi.fn() }));vi.mock("../../../services/i18n", () => ({ t: (key: string) => key }));// keyboard_actions asks the server for its shortcuts at import.-vi.mock("../../../services/server", () => ({ default: { post: h.post, get: async () => [] } }));+vi.mock("../../../services/server", () => ({+ default: { post: h.post, put: h.put, get: async () => [] }+}));/** Local time, so that the clock times the spec expects hold in every time zone. */const STARTED_AT = "2026-10-13T14:29:55";@@ -379,4 +382,31 @@ describe("TranscriptView", () => {await addTurn();expect(scroller.scrollTop).toBe(0);});++ it("hides filler words and stutters while the option is on, and saves the switch", async () => {+ options.set("transcriptCleanupEnabled", "true");+ const content = transcript([+ { source: "system", offsetMs: 0, text: "Uh the the data is old." },+ { source: "system", offsetMs: 1000, text: "Uh." },+ { source: "system", offsetMs: 2000, text: "I know, I know." }+ ]);+ const attachment = { attachmentId: "transcript1", getBlob: async () => ({ content }) };+ const container = renderInto(+ <ParentComponent.Provider value={new Component()}>+ <TranscriptView attachment={attachment as unknown as FAttachment} />+ </ParentComponent.Provider>+ );+ const spans = () => [ ...container.querySelectorAll(".transcript-turn p span") ]+ .map((span) => span.textContent);+ await vi.waitFor(() => expect(spans()).toHaveLength(2));+ expect(spans()).toEqual([ "The data is old.", " I know, I know." ]);++ const toggle = container.querySelector<HTMLInputElement>(".transcript-options .switch-toggle");+ expect(toggle?.checked).toBe(true);+ await act(async () => {+ toggle?.dispatchEvent(new Event("input", { bubbles: true }));+ });+ expect(h.put).toHaveBeenCalledWith("options", { transcriptCleanupEnabled: "false" });+ expect(spans()).toEqual([ "Uh the the data is old.", " Uh.", " I know, I know." ]);+ });});diff --git a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsxindex 30e3c5364e..876e3e2118 100644--- a/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx+++ b/apps/client/src/widgets/type_widgets/transcript/TranscriptView.tsx@@ -1,7 +1,7 @@import "./TranscriptView.css";import {- getSpeakerKey, MAX_SPEAKER_NAME_LENGTH, parseTranscript, setSpeakerName,+ cleanUtteranceText, getSpeakerKey, MAX_SPEAKER_NAME_LENGTH, parseTranscript, setSpeakerName,type TranscriptSource, type TranscriptUtterance} from "@triliumnext/commons";import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "preact/hooks";@@ -13,7 +13,8 @@ import { isIMEComposing } from "../../../services/shortcuts";import utils from "../../../services/utils";import { useTranscriptionState } from "../../launch_bar/RecordMeetingButton";import FormTextBox from "../../react/FormTextBox";-import { useStaticTooltip, useTriliumEvent } from "../../react/hooks";+import FormToggle from "../../react/FormToggle";+import { useStaticTooltip, useTriliumEvent, useTriliumOptionBool } from "../../react/hooks";import LinkButton from "../../react/LinkButton";import NoItems from "../../react/NoItems";@@ -30,10 +31,12 @@ const PARAGRAPH_PAUSE_MS = 2000;* Shows a `transcript` attachment as the conversation it records, one paragraph per speaker turn.* Follows the attachment as a running recording rewrites it, shows which speakers the recording* hears before their words are transcribed, and keeps a reader who is at the end there as new- * turns arrive.+ * turns arrive. With `transcriptCleanupEnabled`, shows each utterance through+ * `cleanUtteranceText()` and leaves out the ones it empties; the stored transcript stays verbatim.*/export default function TranscriptView({ attachment }: { attachment: FAttachment }) {const [ content, setContent ] = useState<string | null>();+ const [ isCleanedUp, setCleanedUp ] = useTriliumOptionBool("transcriptCleanupEnabled");const recording = useTranscriptionState();const pending = recording.noteId === attachment.ownerId ? recording.pendingSources : [];const scrollerRef = useRef<HTMLDivElement>(null);@@ -87,7 +90,14 @@ export default function TranscriptView({ attachment }: { attachment: FAttachmentinitialCountRef.current ??= transcript.utterances.length;const arrived = new Set(transcript.utterances.slice(initialCountRef.current));const names = transcript.speakers ?? {};- const turns = groupIntoTurns(transcript.utterances, names);+ const shownTexts = new Map<TranscriptUtterance, string>();+ for (const utterance of transcript.utterances) {+ const text = isCleanedUp ? cleanUtteranceText(utterance.text) : utterance.text;+ if (text) {+ shownTexts.set(utterance, text);+ }+ }+ const turns = groupIntoTurns([ ...shownTexts.keys() ], names);const lastTurn = turns.at(-1);const speaking = (<p className="transcript-speaking">@@ -97,6 +107,18 @@ export default function TranscriptView({ attachment }: { attachment: FAttachment);return (<div ref={scrollerRef} className="transcript">+ {transcript.utterances.length > 0 && (+ <div className="transcript-options">+ <FormToggle+ switchOnName={t("transcript.cleanup")}+ switchOffName={t("transcript.cleanup")}+ switchOnTooltip={t("transcript.cleanup_tooltip")}+ switchOffTooltip={t("transcript.cleanup_tooltip")}+ currentValue={isCleanedUp}+ onChange={setCleanedUp}+ />+ </div>+ )}{turns.map((turn) => (<div key={turn.offsetMs} className={`transcript-turn transcript-${turn.source}`}><div className="transcript-turn-header">@@ -110,14 +132,17 @@ export default function TranscriptView({ attachment }: { attachment: FAttachment</div>{turn.paragraphs.map((paragraph, index) => (<p key={index}>- {paragraph.map((utterance, position) => (- <span- key={utterance.offsetMs}- className={arrived.has(utterance) ? "transcript-new" : undefined}- >- {position ? ` ${utterance.text}` : utterance.text}- </span>- ))}+ {paragraph.map((utterance, position) => {+ const text = shownTexts.get(utterance) ?? utterance.text;+ return (+ <span+ key={utterance.offsetMs}+ className={arrived.has(utterance) ? "transcript-new" : undefined}+ >+ {position ? ` ${text}` : text}+ </span>+ );+ })}</p>))}{turn === lastTurn && pending.includes(turn.source) && speaking}diff --git a/packages/commons/src/lib/options_interface.ts b/packages/commons/src/lib/options_interface.tsindex 44847581bc..fa2351da52 100644--- a/packages/commons/src/lib/options_interface.ts+++ b/packages/commons/src/lib/options_interface.ts@@ -202,6 +202,8 @@ export interface OptionDefinitions extends KeyboardShortcutsOptions<KeyboardActiautoFixConsistencyIssues: boolean;vimKeymapEnabled: boolean;codeLineWrapEnabled: boolean;+ /** Whether a meeting transcript is shown without filler words and stutters. */+ transcriptCleanupEnabled: boolean;leftPaneVisible: boolean;rightPaneVisible: boolean;nativeTitleBarVisible: boolean;diff --git a/packages/commons/src/lib/transcript.spec.ts b/packages/commons/src/lib/transcript.spec.tsindex bf975e04f9..ba394ec15c 100644--- a/packages/commons/src/lib/transcript.spec.ts+++ b/packages/commons/src/lib/transcript.spec.ts@@ -1,7 +1,8 @@import { describe, expect, it } from "vitest";import {- getSpeakerKey, MAX_SPEAKER_NAME_LENGTH, parseTranscript, setSpeakerName, type Transcript+ cleanUtteranceText, getSpeakerKey, MAX_SPEAKER_NAME_LENGTH, parseTranscript, setSpeakerName,+ type Transcript} from "./transcript.js";const TRANSCRIPT: Transcript = {@@ -39,4 +40,35 @@ describe("transcript", () => {microphone: "x".repeat(MAX_SPEAKER_NAME_LENGTH)});});++ it("removes filler words, keeping the sentence ends and capitals they carried", () => {+ expect(cleanUtteranceText("Uh the data is a bit old.")).toBe("The data is a bit old.");+ expect(cleanUtteranceText("a good point, uh, because uh.")).toBe("a good point, because.");+ expect(cleanUtteranceText("So okay. um so we can go")).toBe("So okay. So we can go");+ expect(cleanUtteranceText("Oh, okay. Hmm, sure.")).toBe("Oh, okay. Hmm, sure.");+ expect(cleanUtteranceText("Uh.")).toBe("");+ });++ it("collapses stutters and restarts, an abandoned word's first letter included", () => {+ expect(cleanUtteranceText("So the the that's a point")).toBe("So the that's a point");+ expect(cleanUtteranceText("we have the uh the memory")).toBe("we have the memory");+ expect(cleanUtteranceText("that was on the on the board")).toBe("that was on the board");+ expect(cleanUtteranceText("I wanted to to to share")).toBe("I wanted to share");+ expect(cleanUtteranceText("Because it's d as you can see")).toBe("Because it's as you can see");+ expect(cleanUtteranceText("probably compare it")).toBe("probably compare it");+ });++ it("keeps meant repeats: numbers, emphasis, commas, grammatical doubles, acronyms", () => {+ for (const meant of [+ "Two two forty, five five fifty.",+ "Okay, okay, okay. Yes yes.",+ "I know, I know.",+ "to check this, this, this, this.",+ "something like that that has a lot",+ "You do it it doesn't grow",+ "the S D K and a PV2"+ ]) {+ expect(cleanUtteranceText(meant)).toBe(meant);+ }+ });});diff --git a/packages/commons/src/lib/transcript.ts b/packages/commons/src/lib/transcript.tsindex 9f3f00b183..4d865106e1 100644--- a/packages/commons/src/lib/transcript.ts+++ b/packages/commons/src/lib/transcript.ts@@ -67,6 +67,58 @@ export function getSpeakerKey(source: TranscriptSource, speaker?: number) {return speaker === undefined ? null : `system:${speaker}`;}+/** Hesitations removed by {@link cleanUtteranceText}. "oh" and "hmm" carry meaning and stay. */+const FILLER_WORDS = new Set([ "uh", "um", "uhm", "er", "erm", "ah", "eh" ]);++/** English words a speaker stutters on. A repeat of any other word is kept. */+const STUTTER_WORDS = new Set([+ "the", "a", "an", "i", "to", "we", "you", "it", "it's", "that", "this", "on", "in", "and",+ "so", "he", "she", "they", "is", "of", "for", "with", "at", "but", "if", "my", "our", "your",+ "their", "be", "are", "was", "can", "will", "do", "have", "has", "what", "how", "when",+ "where", "i'm", "we're", "you're", "there", "then", "or", "as", "by", "from", "his", "her",+ "its", "me", "us"+]);++/** Stutter words that are often grammatical when doubled ("like that that has"). */+const GRAMMATICAL_DOUBLES = new Set([ "that", "it" ]);++/** The longest repeated phrase that {@link cleanUtteranceText} collapses, in words. */+const MAX_RESTART_WORDS = 3;++const NUMBER_WORD = /^(\d|zero$|one$|two$|three$|four$|five$|six$|seven$|eight$|nine$|ten$)/;++interface Word {+ text: string;+ /** The word lowercased, without surrounding punctuation. */+ key: string;+}++/**+ * Removes disfluencies from the text of one utterance: filler words ("uh", "um"), lowercase+ * single-letter fragments of an abandoned word, and restarts, an immediate repeat of one to three+ * words that begins with a stutter word ("the the", "on the on the"). Numbers ("two two forty"),+ * repeats separated by a comma ("I know, I know") and other repeated words ("okay okay") are kept.+ * Returns an empty string when nothing but disfluencies was said.+ */+export function cleanUtteranceText(text: string): string {+ const startsCapitalized = /^\p{Lu}/u.test(text);+ let words: Word[] = text.split(/\s+/).filter(Boolean)+ .map((word) => ({ text: word, key: toKey(word) }));++ words = removeFillers(words);+ words = words.filter((word) => !/^[b-hj-z][,.]?$/.test(word.text));+ collapseRestarts(words);++ let cleaned = words.map((word) => word.text).join(" ")+ .replace(/^[,;:]\s*/, "")+ .replace(/,(\s*[,.?!])/g, "$1")+ .replace(/([.?!]\s+)(\p{Ll})/gu, (_, end: string, letter: string) => end + letter.toUpperCase());+ if (startsCapitalized) {+ cleaned = cleaned.charAt(0).toUpperCase() + cleaned.slice(1);+ }+ return cleaned;+}+/*** Names the speaker of `key`, or gives back its default name when `name` is blank. Returns a new* transcript; the utterances are shared.@@ -80,3 +132,49 @@ export function setSpeakerName(transcript: Transcript, key: string, name: string}return { ...transcript, speakers };}++/** Drops filler words; the sentence end a filler carried ("…because uh.") moves to the word before. */+function removeFillers(words: Word[]) {+ const kept: Word[] = [];+ for (const word of words) {+ if (!FILLER_WORDS.has(word.key)) {+ kept.push(word);+ continue;+ }+ const sentenceEnd = /[.?!]$/.exec(word.text)?.[0];+ const previous = kept.at(-1);+ if (sentenceEnd && previous && !/[.?!]$/.test(previous.text)) {+ previous.text = previous.text.replace(/[,;:]$/, "") + sentenceEnd;+ }+ }+ return kept;+}++/** Removes the first copy of each immediately repeated phrase, longest phrases first. */+function collapseRestarts(words: Word[]) {+ for (let size = MAX_RESTART_WORDS; size >= 1; size--) {+ let start = 0;+ while (start + 2 * size <= words.length) {+ if (isRestart(words, start, size)) {+ words.splice(start, size);+ } else {+ start++;+ }+ }+ }+}++function isRestart(words: Word[], start: number, size: number) {+ const first = words.slice(start, start + size);+ const second = words.slice(start + size, start + 2 * size);+ const lastOfFirst = first[first.length - 1];+ return STUTTER_WORDS.has(first[0].key)+ && !(size === 1 && GRAMMATICAL_DOUBLES.has(first[0].key))+ && !first.some((word) => NUMBER_WORD.test(word.key))+ && !/[,;:.?!]$/.test(lastOfFirst.text)+ && first.every((word, index) => word.key === second[index].key);+}++function toKey(word: string) {+ return word.toLowerCase().replace(/’/g, "'").replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}'-]+$/gu, "");+}diff --git a/packages/trilium-core/src/routes/api/options.ts b/packages/trilium-core/src/routes/api/options.tsindex e9fb037f99..64b7b3524e 100644--- a/packages/trilium-core/src/routes/api/options.ts+++ b/packages/trilium-core/src/routes/api/options.ts@@ -63,6 +63,7 @@ const ALLOWED_OPTIONS = new Set<OptionNames>(["openNoteContexts","vimKeymapEnabled","codeLineWrapEnabled",+ "transcriptCleanupEnabled","codeNotesMimeTypes","contentManagerSortOrder","contentManagerViewMode",diff --git a/packages/trilium-core/src/services/options_init.ts b/packages/trilium-core/src/services/options_init.tsindex 5416d7c17c..5054dd9f6f 100644--- a/packages/trilium-core/src/services/options_init.ts+++ b/packages/trilium-core/src/services/options_init.ts@@ -183,6 +183,7 @@ const defaultOptions: DefaultOption[] = [{ name: "autoFixConsistencyIssues", value: "true", isSynced: false },{ name: "vimKeymapEnabled", value: "false", isSynced: false },{ name: "codeLineWrapEnabled", value: "true", isSynced: false },+ { name: "transcriptCleanupEnabled", value: "true", isSynced: true },{ name: "codeNoteTabWidth", value: "4", isSynced: true },{ name: "codeNoteIndentWithTabs", value: "false", isSynced: true },{