Skip to content

Build your personal knowledge base with Trilium Notes

fix(transcript): trim a word's punctuation in linear time

Elian Doran420e0bbbfe3f
diff --git a/packages/commons/src/lib/transcript.spec.ts b/packages/commons/src/lib/transcript.spec.ts
index ba394ec15c..e48e727717 100644
--- a/packages/commons/src/lib/transcript.spec.ts
+++ b/packages/commons/src/lib/transcript.spec.ts
@@ -71,4 +71,14 @@ describe("transcript", () => {
expect(cleanUtteranceText(meant)).toBe(meant);
}
});
+
+ it("matches words through the punctuation around them, in time linear in their length", () => {
+ expect(cleanUtteranceText("So “uh” (um) we’re we're done")).toBe("So we're done");
+ expect(cleanUtteranceText("And (the) the plan")).toBe("And the plan");
+
+ const punctuated = `a${"&".repeat(100_000)}a`;
+ const startedAt = performance.now();
+ expect(cleanUtteranceText(punctuated)).toBe(punctuated);
+ expect(performance.now() - startedAt).toBeLessThan(250);
+ });
});
diff --git a/packages/commons/src/lib/transcript.ts b/packages/commons/src/lib/transcript.ts
index 4d865106e1..386e565de0 100644
--- a/packages/commons/src/lib/transcript.ts
+++ b/packages/commons/src/lib/transcript.ts
@@ -85,6 +85,11 @@ const GRAMMATICAL_DOUBLES = new Set([ "that", "it" ]);
/** The longest repeated phrase that {@link cleanUtteranceText} collapses, in words. */
const MAX_RESTART_WORDS = 3;
+const LETTER_OR_DIGIT = /^[\p{L}\p{N}]$/u;
+
+/** Characters {@link toKey} keeps at the end of a word, as in "goin'" or "well-". */
+const KEPT_AT_END = "'-";
+
const NUMBER_WORD = /^(\d|zero$|one$|two$|three$|four$|five$|six$|seven$|eight$|nine$|ten$)/;
interface Word {
@@ -175,6 +180,17 @@ function isRestart(words: Word[], start: number, size: number) {
&& first.every((word, index) => word.key === second[index].key);
}
+/** Lowercases `word` and trims the punctuation around it, keeping a trailing `'` or `-`. */
function toKey(word: string) {
- return word.toLowerCase().replace(/’/g, "'").replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}'-]+$/gu, "");
+ const chars = [ ...word.toLowerCase().replace(/’/g, "'") ];
+ let start = 0;
+ let end = chars.length;
+ while (start < end && !LETTER_OR_DIGIT.test(chars[start])) {
+ start++;
+ }
+ while (end > start && !LETTER_OR_DIGIT.test(chars[end - 1])
+ && !KEPT_AT_END.includes(chars[end - 1])) {
+ end--;
+ }
+ return chars.slice(start, end).join("");
}