Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(pdf): list free-hand highlights and drawings in the annotations sidebar
Elian Doran53a063f6a06e
diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.jsonindex 17243d009a..c2cb288662 100644--- a/apps/client/src/translations/en/translation.json+++ b/apps/client/src/translations/en/translation.json@@ -3612,7 +3612,10 @@"pages_alt": "Page {{pageNumber}}","pages_loading": "Loading...","annotations_one": "{{count}} annotation",- "annotations_other": "{{count}} annotations"+ "annotations_other": "{{count}} annotations",+ "annotation_drawing": "Drawing on page {{pageNumber}}",+ "annotation_highlight": "Highlight on page {{pageNumber}}",+ "annotation_note": "Note on page {{pageNumber}}"},"platform_indicator": {"available_on": "Available on {{platform}}"diff --git a/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.css b/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.cssindex 67473e2252..415f83ae85 100644--- a/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.css+++ b/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.css@@ -28,11 +28,26 @@font-size: 18px;}+/* Rows tinted with a dark annotation colour (the pen defaults to black) invert their text. */+.pdf-annotation-item.on-dark,+.pdf-annotation-item.on-dark .tn-icon,+.pdf-annotation-item.on-dark .pdf-annotation-untitled,+.pdf-annotation-item.on-dark .pdf-annotation-highlighted-text,+.pdf-annotation-item.on-dark .pdf-annotation-contents,+.pdf-annotation-item.on-dark .pdf-annotation-author {+ color: #f0f0f0;+}+.pdf-annotation-info {flex: 1;min-width: 0;}+.pdf-annotation-untitled {+ font-size: 13px;+ color: #555;+}+.pdf-annotation-highlighted-text {font-size: 13px;font-style: italic;diff --git a/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.spec.tsx b/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.spec.tsxnew file mode 100644index 0000000000..92c2793288--- /dev/null+++ b/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.spec.tsx@@ -0,0 +1,116 @@+import { render } from "preact";+import { afterEach, describe, expect, it, vi } from "vitest";++import FNote from "../../../entities/fnote";+import options from "../../../services/options";+import PdfAnnotations, { isDark } from "./PdfAnnotations";++// The panel reads the note being displayed and the annotations the viewer iframe published; both+// arrive through hooks that need the whole app context, so they are handed over directly here.+const shown = vi.hoisted(() => ({+ note: null as FNote | null,+ annotations: null as { annotations: PdfAnnotationInfo[]; scrollToAnnotation: () => void } | null+}));+// i18next is not initialised for client specs, so t() would render every label as an empty+// string. Echoing the key and its interpolations instead keeps the assertions about which+// label the panel picks, rather than about the English wording.+vi.mock("../../../services/i18n", () => ({+ t: (key: string, options?: Record<string, unknown>) => `${key}(${JSON.stringify(options ?? {})})`+}));++vi.mock("../../react/hooks", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../../react/hooks")>()),+ useActiveNoteContext: () => ({ note: shown.note }),+ useNoteProperty: (_note: FNote, property: string) => (shown.note as any)?.[property],+ useGetContextData: () => shown.annotations+}));++/** A `file` note holding a PDF — the only combination the panel renders for. */+function pdfNote() {+ return { type: "file", mime: "application/pdf" } as unknown as FNote;+}++function annotation(overrides: Partial<PdfAnnotationInfo>): PdfAnnotationInfo {+ return {+ id: "1R",+ type: "highlight",+ contents: "",+ highlightedText: "",+ author: "",+ pageNumber: 1,+ color: null,+ creationDate: null,+ modificationDate: null,+ ...overrides+ };+}++function renderPanel(annotations: PdfAnnotationInfo[]) {+ // The surrounding RightPanelWidget reads which panels the user collapsed from the options.+ options.set("rightPaneCollapsedItems", JSON.stringify([]));++ const container = document.createElement("div");+ document.body.append(container);+ shown.note = pdfNote();+ shown.annotations = { annotations, scrollToAnnotation: () => {} };+ render(<PdfAnnotations />, container);+ return container;+}++describe("PdfAnnotations", () => {+ afterEach(() => {+ document.body.innerHTML = "";+ shown.note = null;+ shown.annotations = null;+ });++ it("names annotations that carry no text of their own by kind and page", () => {+ // A free-hand highlight is stored as ink and never has text; before #11059 it was+ // dropped, taking the whole panel with it when nothing else was annotated.+ const container = renderPanel([+ annotation({ id: "18R", type: "ink", pageNumber: 4 }),+ annotation({ id: "17R", type: "highlight", pageNumber: 2 }),+ annotation({ id: "5R", highlightedText: "quoted words", contents: "A remark" })+ ]);++ const rows = [ ...container.querySelectorAll(".pdf-annotation-item") ];+ expect(rows).toHaveLength(3);+ expect(rows[0].querySelector(".pdf-annotation-untitled")?.textContent)+ .toBe(`pdf.annotation_drawing({"pageNumber":4})`);+ expect(rows[1].querySelector(".pdf-annotation-untitled")?.textContent)+ .toBe(`pdf.annotation_highlight({"pageNumber":2})`);+ // An annotation with text of its own is described by that text, not by its kind.+ expect(rows[2].textContent).toContain("quoted words");+ expect(rows[2].textContent).toContain("A remark");+ expect(rows[2].querySelector(".pdf-annotation-untitled")).toBeNull();+ });++ it("flips the row's text to light on a dark annotation colour", () => {+ // The pen draws in black by default, and the row is tinted with the annotation's own+ // colour — the default dark text would be unreadable on it.+ const container = renderPanel([+ annotation({ id: "18R", type: "ink", color: "#000000" }),+ annotation({ id: "5R", color: "#ffff98" })+ ]);++ const rows = [ ...container.querySelectorAll(".pdf-annotation-item") ];+ expect(rows[0].classList.contains("on-dark")).toBe(true);+ expect(rows[1].classList.contains("on-dark")).toBe(false);+ });+});++describe("isDark", () => {+ it("separates colours needing light text from the rest", () => {+ expect(isDark("#000000")).toBe(true);+ expect(isDark("#1a3d7c")).toBe(true);+ expect(isDark("#ffff98")).toBe(false);+ expect(isDark("#ffffff")).toBe(false);+ // Green weighs heaviest in the luma formula, so a saturated green reads as light.+ expect(isDark("#00ff00")).toBe(false);+ });++ it("treats a missing or unparseable colour as light, matching the untinted row", () => {+ expect(isDark(null)).toBe(false);+ expect(isDark("rgb(0, 0, 0)")).toBe(false);+ });+});diff --git a/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.tsx b/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.tsxindex 73d9f79f6f..7554b8ef96 100644--- a/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.tsx+++ b/apps/client/src/widgets/sidebar/pdf/PdfAnnotations.tsx@@ -1,5 +1,7 @@import "./PdfAnnotations.css";+import clsx from "clsx";+import { t } from "../../../services/i18n";import { useActiveNoteContext, useGetContextData, useNoteProperty } from "../../react/hooks";import Icon from "../../react/Icon";@@ -8,6 +10,7 @@ import RightPanelWidget from "../RightPanelWidget";const TYPE_ICONS: Record<string, string> = {text: "bx bxs-comment-detail",highlight: "bx bx-highlight",+ ink: "bx bx-pen",};export default function PdfAnnotations() {@@ -49,15 +52,21 @@ function PdfAnnotationItem({const icon = annotation.contents? "bx bxs-comment-detail": TYPE_ICONS[annotation.type] ?? "bx bx-comment";+ // A drawing has no text of its own, and a highlight only gets one where the page has+ // extractable glyphs under it — name those by what they are so the row stays clickable.+ const hasText = !!(annotation.highlightedText || annotation.contents);return (<div- className="pdf-annotation-item"+ className={clsx("pdf-annotation-item", isDark(annotation.color) && "on-dark")}onClick={() => onNavigate(annotation.id, annotation.pageNumber)}style={annotation.color ? { backgroundColor: annotation.color } : undefined}><Icon icon={icon} /><div className="pdf-annotation-info">+ {!hasText && (+ <div className="pdf-annotation-untitled">{describeUntitled(annotation)}</div>+ )}{annotation.highlightedText && (<div className="pdf-annotation-highlighted-text">{annotation.highlightedText}</div>)}@@ -71,3 +80,28 @@ function PdfAnnotationItem({</div>);}++/**+ * Whether a row tinted with this annotation colour needs light text. Highlights are pastel, but+ * the pen draws in black by default, and the row's dark default text would vanish on it.+ */+export function isDark(color: string | null) {+ const rgb = /^#([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i.exec(color ?? "");+ if (!rgb) return false;++ const [ r, g, b ] = rgb.slice(1).map((component) => parseInt(component, 16));+ // Rec. 601 luma, the usual stand-in for perceived brightness.+ return (0.299 * r + 0.587 * g + 0.114 * b) < 140;+}++/** Names an annotation that carries no text of its own, by kind and page. */+function describeUntitled({ type, pageNumber }: PdfAnnotationInfo) {+ switch (type) {+ case "ink":+ return t("pdf.annotation_drawing", { pageNumber });+ case "highlight":+ return t("pdf.annotation_highlight", { pageNumber });+ default:+ return t("pdf.annotation_note", { pageNumber });+ }+}diff --git a/packages/pdfjs-viewer/src/annotations.spec.ts b/packages/pdfjs-viewer/src/annotations.spec.tsindex b789d6e0a1..96ade4c488 100644--- a/packages/pdfjs-viewer/src/annotations.spec.ts+++ b/packages/pdfjs-viewer/src/annotations.spec.ts@@ -59,12 +59,30 @@ describe("processAnnotation", () => {expect(processAnnotation(withAuthor, 1)!.author).toBe("John Doe");});- it("skips non-comment types and empty annotations", () => {+ it("skips types the sidebar does not list", () => {const link = { ...SAMPLE_HIGHLIGHT, annotationType: 2 }; // LINKexpect(processAnnotation(link, 1)).toBeNull();- const empty = { ...SAMPLE_HIGHLIGHT, contentsObj: { str: "", dir: "ltr" }, overlaidText: "" };- expect(processAnnotation(empty, 1)).toBeNull();+ const freeText = { ...SAMPLE_HIGHLIGHT, annotationType: 3 }; // FREETEXT+ expect(processAnnotation(freeText, 1)).toBeNull();+ });++ it("keeps an annotation carrying no text at all", () => {+ // pdf.js writes a free-hand highlight as Ink, which never has contents or overlaidText;+ // dropping those hid every highlight not drawn over selected text (#11059).+ const drawing = {+ ...SAMPLE_HIGHLIGHT,+ annotationType: AnnotationType.INK,+ contentsObj: { str: "", dir: "ltr" },+ overlaidText: null+ };++ const result = processAnnotation(drawing, 4)!;+ expect(result).not.toBeNull();+ expect(result.type).toBe("ink");+ expect(result.contents).toBe("");+ expect(result.highlightedText).toBe("");+ expect(result.pageNumber).toBe(4);});it("keeps annotations with only one of contents or highlightedText", () => {@@ -101,7 +119,7 @@ describe("extraction from a real document", () => {afterEach(() => uninstallViewerApp());- /** The two annotations the fixture expects to surface, in page order. */+ /** The annotations the fixture expects to surface, in page order. */const EXPECTED = [expect.objectContaining({id: "5R",@@ -118,16 +136,31 @@ describe("extraction from a real document", () => {author: "Bob",pageNumber: 1,color: "#0000ff"+ }),+ expect.objectContaining({+ id: "17R",+ type: "highlight",+ contents: "",+ highlightedText: "",+ pageNumber: 2+ }),+ expect.objectContaining({+ id: "18R",+ type: "ink",+ contents: "",+ highlightedText: "",+ pageNumber: 2,+ color: "#000000"})];- it("walks every page and keeps only the annotations worth showing", async () => {+ it("walks every page and lists every annotation kind the sidebar shows", async () => {viewer = await installViewerApp(allFeaturesPdf());await setupPdfAnnotations();- // Page 2 holds a link (wrong type) and a highlight with no comment or highlighted- // text; neither should reach the sidebar, and the page loop still has to visit it.+ // Page 2's link is the only annotation filtered out — the highlight with nothing to+ // show and the ink drawing both belong in the sidebar, named by kind (#11059).expect(viewer.lastMessageOfType("pdfjs-viewer-annotations").annotations).toEqual(EXPECTED);});@@ -165,7 +198,7 @@ describe("extraction from a real document", () => {await setupPdfAnnotations();const { annotations } = viewer.lastMessageOfType("pdfjs-viewer-annotations");- expect(annotations.map((annotation: any) => annotation.id)).toEqual([ "14R" ]);+ expect(annotations.map((annotation: any) => annotation.id)).toEqual([ "14R", "17R", "18R" ]);});it("reports an empty list when extraction fails", async () => {@@ -213,7 +246,12 @@ describe("re-extraction from saved bytes", () => {// Saving and reopening must not lose the annotations the sidebar is showing.expect(viewer.lastMessageOfType("pdfjs-viewer-annotations").annotations)- .toEqual([ expect.objectContaining({ contents: "A remark" }), expect.objectContaining({ contents: "A sticky note" }) ]);+ .toEqual([+ expect.objectContaining({ contents: "A remark" }),+ expect.objectContaining({ contents: "A sticky note" }),+ expect.objectContaining({ id: "17R", type: "highlight" }),+ expect.objectContaining({ id: "18R", type: "ink" })+ ]);// The temporary document owns a worker, so failing to destroy it leaks one per save.expect(destroy).toHaveBeenCalledTimes(1);});diff --git a/packages/pdfjs-viewer/src/annotations.ts b/packages/pdfjs-viewer/src/annotations.tsindex 909eaf224c..559f19fe1a 100644--- a/packages/pdfjs-viewer/src/annotations.ts+++ b/packages/pdfjs-viewer/src/annotations.ts@@ -2,17 +2,25 @@export const AnnotationType = {TEXT: 1,HIGHLIGHT: 9,+ INK: 15,} as const;-/** Annotation types we display in the sidebar. */-const COMMENT_TYPES = new Set([+/**+ * Annotation types we display in the sidebar. Ink is here because pdf.js writes a free-hand+ * highlight — one drawn with the highlight tool instead of over selected text — as an Ink+ * annotation, not a Highlight one, so leaving it out hid every highlight not made by selecting+ * text. It also covers the pen tool, whose drawings belong in the list just as much.+ */+const COMMENT_TYPES = new Set<number>([AnnotationType.TEXT,AnnotationType.HIGHLIGHT,+ AnnotationType.INK,]);const TYPE_NAMES: Record<number, string> = {[AnnotationType.TEXT]: "text",[AnnotationType.HIGHLIGHT]: "highlight",+ [AnnotationType.INK]: "ink",};/**@@ -24,14 +32,12 @@ export function processAnnotation(ann: Record<string, any>, pageNumber: number):return null;}+ // Both can be empty: a drawing carries no text at all, and pdf.js only fills overlaidText+ // for a highlight whose quadpoints cover extractable glyphs — never on a scanned page. The+ // sidebar labels such an entry by its type, so the annotation is still reachable.const contents = ann.contentsObj?.str || "";const highlightedText = ann.overlaidText || "";- // Skip annotations that have no meaningful content- if (!contents && !highlightedText) {- return null;- }-return {id: ann.id,type: TYPE_NAMES[ann.annotationType] ?? "unknown",diff --git a/packages/pdfjs-viewer/src/test/fixture_pdf.ts b/packages/pdfjs-viewer/src/test/fixture_pdf.tsindex 7b15d7c716..f988f4c5d6 100644--- a/packages/pdfjs-viewer/src/test/fixture_pdf.ts+++ b/packages/pdfjs-viewer/src/test/fixture_pdf.ts@@ -38,8 +38,8 @@ export function buildPdf(objects: string[]): Uint8Array {export const ATTACHMENT_TEXT = "hello attachment payload";/**- * The all-features fixture: two pages, a two-level outline, a highlight plus a text- * annotation, an embedded attachment, and two optional-content groups (one hidden).+ * The all-features fixture: two pages, a two-level outline, a highlight plus a text, an ink+ * and a link annotation, an embedded attachment, and two optional-content groups (one hidden).** Kept as a single document so one `getDocument()` call serves every contract test; the* per-feature expectations live in the specs.@@ -55,9 +55,9 @@ export function allFeaturesPdf(): Uint8Array {"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>",// 3 — page 1, carrying both annotations."<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Annots [5 0 R 14 0 R] >>",- // 4 — page 2, target of the second outline entry, carrying the two annotations that- // must be filtered out: a link (wrong type) and a highlight with nothing to show.- "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Annots [16 0 R 17 0 R] >>",+ // 4 — page 2, target of the second outline entry, carrying a link (the one type the+ // sidebar filters out) alongside the two textless annotations it still lists.+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Annots [16 0 R 17 0 R 18 0 R] >>",// 5 — highlight annotation with a comment, author and colour."<< /Type /Annot /Subtype /Highlight /Rect [100 700 200 720] "+ "/Contents (A remark) /T (Alice) /C [1 0 0] "@@ -87,9 +87,13 @@ export function allFeaturesPdf(): Uint8Array {+ `stream\n${ATTACHMENT_TEXT}\nendstream`,// 16 — a link annotation: a type the sidebar does not display."<< /Type /Annot /Subtype /Link /Rect [50 50 150 70] /A << /S /URI /URI (https://example.com) >> >>",- // 17 — a highlight with neither a comment nor highlighted text, so nothing to show.+ // 17 — a highlight with neither a comment nor highlighted text: what a highlight on a+ // page with no extractable glyphs (a scan) looks like."<< /Type /Annot /Subtype /Highlight /Rect [200 200 300 220] "- + "/QuadPoints [200 220 300 220 200 200 300 200] >>"+ + "/QuadPoints [200 220 300 220 200 200 300 200] >>",+ // 18 — an ink annotation, which is what pdf.js writes for a free-hand highlight.+ "<< /Type /Annot /Subtype /Ink /Rect [400 400 500 450] /C [0 0 0] "+ + "/InkList [[400 400 450 450 500 400]] >>"]);}diff --git a/packages/pdfjs-viewer/src/upstream_contract.spec.ts b/packages/pdfjs-viewer/src/upstream_contract.spec.tsindex 77bcb4cf77..704bc11d1b 100644--- a/packages/pdfjs-viewer/src/upstream_contract.spec.ts+++ b/packages/pdfjs-viewer/src/upstream_contract.spec.ts@@ -44,6 +44,7 @@ describe("annotation type constants", () => {// bundle owns pdf.js at runtime; that copy has to be kept honest from here.expect(AnnotationType.TEXT).toBe(UpstreamAnnotationType.TEXT);expect(AnnotationType.HIGHLIGHT).toBe(UpstreamAnnotationType.HIGHLIGHT);+ expect(AnnotationType.INK).toBe(UpstreamAnnotationType.INK);// editing.ts documents this as "Matches AnnotationEditorType.NONE in pdf.js".expect(ANNOTATION_EDITOR_MODE_NONE).toBe(AnnotationEditorType.NONE);