Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
feat(clipper): shrink a screenshot by its width only
Elian Doran2c867982560c
diff --git a/apps/server/src/assets/doc_notes/en/User Guide/User Guide/Installation & Setup/Web Clipper.html b/apps/server/src/assets/doc_notes/en/User Guide/User Guide/Installation & Setup/Web Clipper.htmlindex 2418dc2c34..539462fd67 100644--- a/apps/server/src/assets/doc_notes/en/User Guide/User Guide/Installation & Setup/Web Clipper.html+++ b/apps/server/src/assets/doc_notes/en/User Guide/User Guide/Installation & Setup/Web Clipper.html@@ -118,7 +118,8 @@a background that fills the whole window repeats on every screen. A longpage is saved as several images, one below the other in the note. A pagelonger than 100 screens is cut off there, and the message after saving- says so.</p>+ says so. Screenshots keep their full height in Trilium: <em>Resize image</em> (under <em>Automatically compress images</em> in+ Options → Media) limits only their width.</p></li><li><p>The popup stays open while Trilium saves, which can take a few secondsdiff --git a/apps/server/src/routes/api/clipper.spec.ts b/apps/server/src/routes/api/clipper.spec.tsindex 6065e8c715..222400bf79 100644--- a/apps/server/src/routes/api/clipper.spec.ts+++ b/apps/server/src/routes/api/clipper.spec.ts@@ -8,6 +8,7 @@ const { buildNote } = becca_easy_mocking;const resolveYouTubePreview = vi.hoisted(() => vi.fn());const storeFavicon = vi.hoisted(() => vi.fn());+const saveImageToAttachment = vi.hoisted(() => vi.fn(() => ({ attachmentId: "foo", title: "encodedTitle" })));vi.mock("@triliumnext/core", async (importOriginal) => ({...await importOriginal<typeof import("@triliumnext/core")>(),@@ -17,12 +18,7 @@ vi.mock("@triliumnext/core", async (importOriginal) => ({vi.mock("../../services/image.js", () => ({default: {- saveImageToAttachment() {- return {- attachmentId: "foo",- title: "encodedTitle",- };- },+ saveImageToAttachment,// The clipping is read as soon as this answers, so the picture has to be stored by// then; nothing here defers, so there is nothing for the wait to do.awaitImageWrite: async () => {}@@ -57,6 +53,18 @@ describe("processContent", () => {expect(processed).toStrictEqual(`<img src="api/attachments/foo/image/encodedTitle" >`);});+ it("shrinks a screenshot by its width alone, and any other image to fit", async () => {+ const PNG = "data:image/png;base64,iVBORw0KGgo=";+ saveImageToAttachment.mockClear();+ await cls.init(() => processContent([+ { imageId: "screenshotImage00000", src: "inline.png", dataUrl: PNG, screenshot: true },+ { imageId: "pictureImage00000000", src: "inline.png", dataUrl: PNG, screenshot: "yes" }+ ], note, `<img src="screenshotImage00000"><img src="pictureImage00000000">`));++ expect(saveImageToAttachment.mock.calls.map((call) => (call as unknown[]).slice(3)))+ .toEqual([ [ "width" ], [ true ] ]);+ });+it("skips over non-data images", async () => {for (const url of [ "foo", "" ]) {const processed = await cls.init(() => processContent(diff --git a/apps/server/src/routes/api/clipper.ts b/apps/server/src/routes/api/clipper.tsindex d193608ea5..8f730bb6ac 100644--- a/apps/server/src/routes/api/clipper.ts+++ b/apps/server/src/routes/api/clipper.ts@@ -18,6 +18,8 @@ interface Image {src: string;dataUrl: string;imageId: string;+ /** Sent by recent versions of the extension; an older one's screenshots are shrunk to fit. */+ screenshot?: unknown;}async function addClipping(req: Request) {@@ -239,7 +241,7 @@ export async function processContent(images: Image[], note: BNote, content: strilet rewrittenContent = sanitize.sanitizeHtml(await replaceYouTubePlayers(note, content));if (images) {- for (const { src, dataUrl, imageId } of images) {+ for (const { src, dataUrl, imageId, screenshot } of images) {const filename = path.basename(src);if (!dataUrl || !dataUrl.startsWith("data:image")) {@@ -251,7 +253,10 @@ export async function processContent(images: Image[], note: BNote, content: striconst buffer = Buffer.from(dataUrl.split(",")[1], "base64");- const attachment = imageService.saveImageToAttachment(note.noteId, buffer, filename, true);+ // A screenshot is held to the width setting only: fitting a long page into it by its+ // height would leave a strip too narrow to read.+ const attachment = imageService.saveImageToAttachment(+ note.noteId, buffer, filename, screenshot === true ? "width" : true);// The URL below goes into the note as an image source, and the note is opened as soon// as this answers. Waiting for the bytes is what stops the clipping from being readdiff --git a/apps/server/src/services/image.ts b/apps/server/src/services/image.tsindex aa256e63fd..f228ea3c3b 100644--- a/apps/server/src/services/image.ts+++ b/apps/server/src/services/image.ts@@ -3,7 +3,7 @@* Re-exports core image service and adds OCR scheduling.*/-import { getLog, imageService, options as optionService } from "@triliumnext/core";+import { getLog, imageService, type ImageShrink, options as optionService } from "@triliumnext/core";import ocrService from "./ocr/ocr_service.js";function scheduleOcrForNote(noteId: string) {@@ -43,7 +43,7 @@ function saveImageToAttachment(noteId: string,uploadBuffer: Uint8Array,originalName: string,- shrinkImageSwitch?: boolean,+ shrinkImageSwitch?: ImageShrink,trimFilename = false) {const result = imageService.saveImageToAttachment(noteId, uploadBuffer, originalName, shrinkImageSwitch, trimFilename);diff --git a/apps/server/src/services/image_codec.ts b/apps/server/src/services/image_codec.tsindex 7a33e03117..fa718be936 100644--- a/apps/server/src/services/image_codec.ts+++ b/apps/server/src/services/image_codec.ts@@ -117,7 +117,9 @@ export function planFromBytes(// to an image should not cost more than doing it. Dimensions the header does not give up — past// the end of a prefix, or behind metadata — leave this open, and the decode settles it.const declared = inspectImage(bytes);- const declaredEdge = Math.max(declared.width ?? 0, declared.height ?? 0);+ const declaredEdge = request.widthOnly+ ? declared.width ?? 0+ : Math.max(declared.width ?? 0, declared.height ?? 0);const mayNeedResize = request.resize && (declaredEdge === 0 || declaredEdge > request.maxWidthHeight);if (!mayNeedResize && !worthReencoding) {@@ -378,7 +380,8 @@ export async function compressImageBytes(const start = Date.now();const image = await decodeImage(buffer, budgetMb);const { width, height } = image.bitmap;- const needsResize = request.resize && Math.max(width, height) > request.maxWidthHeight;+ const needsResize = request.resize+ && (request.widthOnly ? width : Math.max(width, height)) > request.maxWidthHeight;// Only consulted where it changes the answer. JPEG has no alpha channel, so a transparent// image cannot be converted — it is optimized instead, that being the best still available@@ -399,7 +402,7 @@ export async function compressImageBytes(}if (needsResize) {- if (width >= height) {+ if (request.widthOnly || width >= height) {image.resize({ w: request.maxWidthHeight });} else {image.resize({ h: request.maxWidthHeight });diff --git a/apps/server/src/services/image_provider.spec.ts b/apps/server/src/services/image_provider.spec.tsindex 429df3743e..f5102653e4 100644--- a/apps/server/src/services/image_provider.spec.ts+++ b/apps/server/src/services/image_provider.spec.ts@@ -250,6 +250,22 @@ describe('serverImageProvider.processImage', () => {expect(result.buffer.byteLength).toBeLessThan(tallPng.byteLength);});+ it('holds only the width of an image shrunk by width, however tall it is', async () => {+ const sizeOf = async (shrink: boolean | 'width') => {+ const { bitmap } = await Jimp.read(Buffer.from(+ (await serverImageProvider.processImage(tallPng, 'tall.png', shrink)).buffer));+ return [ bitmap.width, bitmap.height ];+ };+ setOptions({ imageMaxWidthHeight: '100' });+ expect(await sizeOf(true)).toEqual([ 33, 100 ]);+ expect(await sizeOf('width')).toEqual([ 100, 300 ]);++ // 200x600 is within a bound of 300 by width, and a PNG kept as it is has nothing else to gain.+ setOptions({ imageMaxWidthHeight: '300', imagePngHandling: 'keep' });+ expect((await serverImageProvider.processImage(tallPng, 'tall.png', 'width')).buffer).toBe(tallPng);+ expect(await sizeOf(true)).toEqual([ 100, 300 ]);+ });+it('keeps the original buffer when shrinking does not reduce the size', async () => {setOptions({ imageMaxWidthHeight: '2000' });diff --git a/apps/server/src/services/image_provider.ts b/apps/server/src/services/image_provider.tsindex fe6d3993b0..66b819937d 100644--- a/apps/server/src/services/image_provider.ts+++ b/apps/server/src/services/image_provider.ts@@ -7,7 +7,7 @@*/import { getLog, imageCompressionService, options as optionService } from "@triliumnext/core";-import type { ImageCompressionOutcome, ImageCompressionPlan, ImageCompressionRequest, ImageFormat, ImageProvider, PreviewResizeOutcome, PreviewResizeRequest, ProcessedImage } from "@triliumnext/core/src/services/image_provider.js";+import type { ImageCompressionOutcome, ImageCompressionPlan, ImageCompressionRequest, ImageFormat, ImageProvider, ImageShrink, PreviewResizeOutcome, PreviewResizeRequest, ProcessedImage } from "@triliumnext/core/src/services/image_provider.js";import { createConcurrencyGate } from "./concurrency_gate.js";import {@@ -73,7 +73,7 @@ export const serverImageProvider: ImageProvider = {* stored untouched — a skip here is not a failure, it is the answer that this image had nothing* to gain, and the original is what should be kept in that case anyway.*/- async processImage(buffer: Uint8Array, originalName: string, shrink: boolean): Promise<ProcessedImage> {+ async processImage(buffer: Uint8Array, originalName: string, shrink: ImageShrink): Promise<ProcessedImage> {const original = async (): Promise<ProcessedImage> => ({buffer,format: (await getImageTypeFromBuffer(buffer)) ?? { ext: "dat", mime: "application/octet-stream" }@@ -84,8 +84,11 @@ export const serverImageProvider: ImageProvider = {}try {- const outcome = await automaticCompression.run(() =>- this.compressImage(buffer, imageCompressionService.automaticCompressionRequest()));+ const request = {+ ...imageCompressionService.automaticCompressionRequest(),+ ...(shrink === "width" && { widthOnly: true })+ };+ const outcome = await automaticCompression.run(() => this.compressImage(buffer, request));return outcome.compressed ? { buffer: outcome.buffer, format: outcome.format } : original();} catch (e: unknown) {diff --git a/apps/standalone/src/services/image_provider.ts b/apps/standalone/src/services/image_provider.tsindex 9703ad0229..7a70dd7da7 100644--- a/apps/standalone/src/services/image_provider.ts+++ b/apps/standalone/src/services/image_provider.ts@@ -4,7 +4,7 @@* Images are saved as-is without resizing.*/-import type { ImageCompressionOutcome, ImageCompressionPlan, ImageFormat, ImageProvider, PreviewResizeOutcome, ProcessedImage } from "@triliumnext/core";+import type { ImageCompressionOutcome, ImageCompressionPlan, ImageFormat, ImageProvider, ImageShrink, PreviewResizeOutcome, ProcessedImage } from "@triliumnext/core";import { inspectImage, UNKNOWN_FORMAT } from "@triliumnext/core/src/services/image_inspect.js";/**@@ -22,7 +22,7 @@ export const standaloneImageProvider: ImageProvider = {return getImageTypeFromBuffer(buffer);},- async processImage(buffer: Uint8Array, _originalName: string, _shrink: boolean): Promise<ProcessedImage> {+ async processImage(buffer: Uint8Array, _originalName: string, _shrink: ImageShrink): Promise<ProcessedImage> {// Standalone doesn't do compression - just detect format and return originalconst format = getImageTypeFromBuffer(buffer) || { ext: "dat", mime: "application/octet-stream" };diff --git a/apps/web-clipper/entrypoints/background/index.spec.ts b/apps/web-clipper/entrypoints/background/index.spec.tsindex 976fc12eac..058faa265f 100644--- a/apps/web-clipper/entrypoints/background/index.spec.ts+++ b/apps/web-clipper/entrypoints/background/index.spec.ts@@ -544,7 +544,7 @@ describe("background", () => {expect(payload).toEqual({title: "Active page",content: expect.stringMatching(/^<img src="[A-Za-z0-9]{20}">$/),- images: [ { imageId: expect.any(String), src: "inline.png", dataUrl: PNG } ],+ images: [ { imageId: expect.any(String), src: "inline.png", dataUrl: PNG, screenshot: true } ],pageUrl: PAGE_URL});const { content, images } = payload as {@@ -594,7 +594,7 @@ describe("background", () => {pageUrl: MENU_URL});expect(lastPayload()).toMatchObject({- images: [ { src: "inline.png", dataUrl: PNG } ],+ images: [ { src: "inline.png", dataUrl: PNG, screenshot: true } ],pageUrl: MENU_URL});expect(await lastToast()).toMatchObject({ noteId: "saved" });@@ -721,8 +721,10 @@ describe("background", () => {await run(sendRuntimeMessage({ name: "save-full-page-screenshot" }));- const { content, images } = lastPayload() as { content: string, images: { imageId: string, dataUrl: string }[] };- expect(images.map((image) => image.dataUrl)).toEqual([ PNG, CROPPED_PNG ]);+ const { content, images } = lastPayload() as {+ content: string, images: { imageId: string, dataUrl: string, screenshot: boolean }[]+ };+ expect(images.map((image) => [ image.dataUrl, image.screenshot ])).toEqual([ [ PNG, true ], [ CROPPED_PNG, true ] ]);expect(content).toBe(images.map((image) => `<img src="${image.imageId}">`).join(""));});@@ -822,6 +824,7 @@ describe("background", () => {} ],pageUrl: PAGE_URL});+ expect((lastPayload() as { images: object[] }).images[0]).not.toHaveProperty("screenshot");expect(await lastToast()).toMatchObject({ noteId: "saved" });fetchMock.mockResolvedValueOnce(new Response("Not found", { status: 404 }));diff --git a/apps/web-clipper/entrypoints/background/index.ts b/apps/web-clipper/entrypoints/background/index.tsindex 457162550f..9aee689235 100644--- a/apps/web-clipper/entrypoints/background/index.ts+++ b/apps/web-clipper/entrypoints/background/index.ts@@ -55,8 +55,11 @@ const MAX_SCREENSHOT_CAPTURES = 100;/** The largest PDF the clipper saves. It reaches Trilium as base64 in a single request. */const MAX_PDF_SIZE = 100 * 1024 * 1024;-/** An image of a clipping; `content` references it by `imageId` until Trilium stores it. */-type ClippedImage = { imageId: string, src: string, dataUrl?: string };+/**+ * An image of a clipping; `content` references it by `imageId` until Trilium stores it. Trilium+ * holds a `screenshot` to the width setting alone, so a long page stays readable.+ */+type ClippedImage = { imageId: string, src: string, dataUrl?: string, screenshot?: boolean };/** `linkText` is only available on Firefox. */type ContextMenuInfo = Browser.contextMenus.OnClickData & { linkText?: string };@@ -449,8 +452,12 @@ export default defineBackground(() => {}/** A clipping of the images, one after another. */- async function getImagePayload(srcs: string[], pageUrl: string | null | undefined) {- const images = srcs.map((src) => ({ imageId: randomString(20), src }));+ async function getImagePayload(srcs: string[], pageUrl: string | null | undefined, screenshot = false) {+ const images: ClippedImage[] = srcs.map((src) => ({+ imageId: randomString(20),+ src,+ ...(screenshot && { screenshot })+ }));const activeTab = await getActiveTab();const { description } = await readPageMetadata();const payload = {@@ -493,7 +500,7 @@ export default defineBackground(() => {pageUrl: string | null | undefined,message = "Screenshot has been saved to Trilium."): Promise<Saved> {- const { payload } = await getImagePayload(srcs, pageUrl);+ const { payload } = await getImagePayload(srcs, pageUrl, true);const resp = await triliumServerFacade.callService("POST", "clippings", payload);diff --git a/docs/User Guide/User Guide/Installation & Setup/Web Clipper.md b/docs/User Guide/User Guide/Installation & Setup/Web Clipper.mdindex 82ab726075..43deb21d71 100644--- a/docs/User Guide/User Guide/Installation & Setup/Web Clipper.md+++ b/docs/User Guide/User Guide/Installation & Setup/Web Clipper.md@@ -44,7 +44,7 @@ Clicking the extension's button in the browser toolbar opens its popup:The popup opens on the selection if there is one, otherwise on _Page_, showing the article, or the whole page when the page has no article. On a PDF, it opens on _PDF_. On a page that shows nothing to save, it opens on _Bookmark_ and says so. The keyboard shortcuts and the context menu still save right away, without a preview: saving the whole page that way saves the article, the whole page when the page has no article, or the PDF when the tab shows one.* Pages the extension cannot access (the browser's own pages, the extension stores, and pages that were open before the extension was installed until they are reloaded) cannot be saved or captured, so the popup says so in place of the preview, and only _Tabs_ remains available there.* Below the preview, a row of buttons offers the other actions: _Crop_ (a screenshot of an area you select), _Screenshot_ (of the visible part of the page), _Full page_ (a screenshot of the whole page) and _Tabs_ (saves the links of every tab in the current window as a list of <a class="reference-link" href="../Note%20Types/Text/Link%20Previews.md">Link Previews</a>, or of plain links when _Autogenerate link previews_ is turned off). Hovering over a button shows its full name and, if it has one, its keyboard shortcut.-* _Full page_ scrolls through the page, takes a screenshot of each part and joins them into one image, then scrolls back to where the page was. The scrollbars are hidden while it does, so they do not appear in the image. Stay on the tab until it is saved: switching to another tab stops the screenshot. A header that stays at the top of the window appears once, at the top of the image, and headings that stick while scrolling appear where they are in the page. Other elements that stay in place while scrolling (such as cookie banners and chat buttons) are left out, while a background that fills the whole window repeats on every screen. A long page is saved as several images, one below the other in the note. A page longer than 100 screens is cut off there, and the message after saving says so.+* _Full page_ scrolls through the page, takes a screenshot of each part and joins them into one image, then scrolls back to where the page was. The scrollbars are hidden while it does, so they do not appear in the image. Stay on the tab until it is saved: switching to another tab stops the screenshot. A header that stays at the top of the window appears once, at the top of the image, and headings that stick while scrolling appear where they are in the page. Other elements that stay in place while scrolling (such as cookie banners and chat buttons) are left out, while a background that fills the whole window repeats on every screen. A long page is saved as several images, one below the other in the note. A page longer than 100 screens is cut off there, and the message after saving says so. Screenshots keep their full height in Trilium: _Resize image_ (under _Automatically compress images_ in Options → Media) limits only their width.* The popup stays open while Trilium saves, which can take a few seconds for a long page or a large PDF, and the other actions wait until it is done. The bar at the top of the popup then says, in green, that the clipping was saved (in place of the notice that the page was already clipped), with _Open note_ to open it in Trilium (and, after _Tabs_, _Close saved tabs_), or, in red, why it could not be saved. The same clipping cannot be saved twice by accident: the save button stays disabled until another mode is chosen. _Crop_ is the exception: the popup closes so that an area of the page can be selected, and the notification on the page reports the outcome.* If the current page was already clipped, a notice at the top offers to open the note in Trilium.* If Trilium cannot be found, the popup explains why instead of showing the actions, with buttons to look for Trilium again and to open the options.diff --git a/packages/trilium-core/src/index.ts b/packages/trilium-core/src/index.tsindex 9760b9b0b2..4393da225f 100644--- a/packages/trilium-core/src/index.ts+++ b/packages/trilium-core/src/index.ts@@ -141,7 +141,7 @@ export { default as sync_mutex } from "./services/sync_mutex";export { default as setup } from "./services/setup";export { getPlatform, type PlatformProvider } from "./services/platform";export { InAppHelpProvider } from "./services/in_app_help";-export { type ImageProvider, type ImageFormat, type ImageCompressionOutcome, type ImageCompressionRequest, type ProcessedImage, getImageProvider } from "./services/image_provider";+export { type ImageProvider, type ImageFormat, type ImageCompressionOutcome, type ImageCompressionRequest, type ImageShrink, type ProcessedImage, getImageProvider } from "./services/image_provider";export { default as imageCompressionService } from "./services/image_compression";export { default as imageInventoryService, type ImageInventoryOptions } from "./services/image_inventory";export { getContentCounts } from "./services/space_usage";diff --git a/packages/trilium-core/src/services/image.ts b/packages/trilium-core/src/services/image.tsindex 1b159b6fe4..9de2f816b8 100644--- a/packages/trilium-core/src/services/image.ts+++ b/packages/trilium-core/src/services/image.ts@@ -9,7 +9,7 @@ import sanitizeFilename from "sanitize-filename";import becca from "../becca/becca.js";import { getContext } from "./context.js";import { getLog } from "./log.js";-import { getImageProvider } from "./image_provider.js";+import { getImageProvider, type ImageShrink } from "./image_provider.js";import noteService from "./notes.js";import protectedSessionService from "./protected_session.js";import { getSql } from "./sql/index.js";@@ -176,7 +176,7 @@ function saveImageToAttachment(noteId: string,uploadBuffer: Uint8Array,originalName: string,- shrinkImageSwitch?: boolean,+ shrinkImageSwitch?: ImageShrink,trimFilename = false,/** Which kind of picture this is; see {@link IMAGE_ATTACHMENT_ROLES}. */role: ImageAttachmentRole = "image"@@ -234,7 +234,7 @@ function saveImageToAttachment(// Process image asynchronouslyconst attachmentId = attachment.attachmentId;- trackWrite(getImageProvider().processImage(uploadBuffer, originalName, !!shrinkImageSwitch).then(({ buffer, format }) => {+ trackWrite(getImageProvider().processImage(uploadBuffer, originalName, shrinkImageSwitch ?? false).then(({ buffer, format }) => {getContext().init(() => {getSql().transactional(() => {if (!attachmentId) {diff --git a/packages/trilium-core/src/services/image_provider.ts b/packages/trilium-core/src/services/image_provider.tsindex 9fc66f3a64..b7ab852630 100644--- a/packages/trilium-core/src/services/image_provider.ts+++ b/packages/trilium-core/src/services/image_provider.ts@@ -11,6 +11,12 @@ export interface ImageFormat {mime: string;}+/**+ * How an arriving image is shrunk: not at all, to fit `imageMaxWidthHeight` on its longest edge,+ * or with that bound on its width alone (`"width"`), which keeps a long screenshot readable.+ */+export type ImageShrink = boolean | "width";+export interface ProcessedImage {buffer: Uint8Array;format: ImageFormat;@@ -25,6 +31,8 @@ export interface ImageCompressionRequest {resize: boolean;/** Longest edge in pixels. Only consulted when {@link resize} is on. */maxWidthHeight: number;+ /** Whether {@link maxWidthHeight} bounds the width alone, so a tall image keeps its height. */+ widthOnly?: boolean;/** What becomes of an already-lossy image (JPEG): left as encoded, or recompressed. */jpegHandling: ImageJpegHandling;/** What becomes of a lossless image (PNG): left alone, quantized in place, or converted. */@@ -49,10 +57,10 @@ export interface ImageProvider {* Process image - may resize/compress depending on implementation.* @param buffer - Raw image data* @param originalName - Original filename for logging- * @param shrink - Whether to attempt shrinking the image+ * @param shrink - Whether to attempt shrinking the image, and by which edge* @returns Processed image buffer and detected format*/- processImage(buffer: Uint8Array, originalName: string, shrink: boolean): Promise<ProcessedImage>;+ processImage(buffer: Uint8Array, originalName: string, shrink: ImageShrink): Promise<ProcessedImage>;/*** Recompress an image the user has explicitly asked to shrink, ignoring the `compressImages`