AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.
HTML ai-agentcloudflarecloudflare-d1cloudflare-r2cloudflare-workersdockerelectronevernoteevernote-alternativeknowledge-basemcpnote-takingobsidian-alternativeoffline-firstpersonal-knowledge-basepwaself-hostedsqlitetiptapweb-clipper
feat(extension): keep page clip images inside the note (#474)
Dariusz Koryto40321fe1f822
diff --git a/apps/extension/README.md b/apps/extension/README.mdindex 38e552f2..c4ecc4c2 100644--- a/apps/extension/README.md+++ b/apps/extension/README.md@@ -8,6 +8,7 @@ Chrome, Edge, and Firefox Manifest V3 extension for saving the current webpage,- Test the connection and select a default notebook.- Click the extension action to capture the selected content, or extract the article body with Mozilla Readability when there is no selection.- Convert the extracted article HTML into Markdown with Turndown before uploading it.+- Copy the images in a saved page or selection into the note itself, so the note still shows them if the source page removes or changes them. The note is saved first; up to 30 images (20 MB each, 60 MB in total) are then uploaded as note attachments and the note is pointed at them. An image the extension cannot read, such as one blocked by its site or in an unsupported format, keeps its original address. If the note is edited before the images finish copying, the edit is kept and the images keep their original addresses. No extra site access is requested for this.- Right-click an image and choose **Save image to EdgeEver**. The image file is uploaded into a new note in the default notebook, with the source page linked underneath. An image saved from Google Search records “Google Search” and the search keyword instead of the results URL.- Select text on a page, right-click, and choose **Save selection to EdgeEver**. That passage is saved as a new note in the default notebook, with the source page linked underneath. The rest of the page is not included.- On X, right-click the post text and choose **Save this post to EdgeEver**. The post text, author, time, and photos already shown in that post are saved into one note. Text folded behind Show more is expanded first, so a long post is saved in full. Right-clicking a photo saves that image instead, so the command stays on the top-level menu. A single-post page can be saved immediately. On a timeline, the extension asks once for access to X so it can remember the post under the pointer.diff --git a/apps/extension/package.json b/apps/extension/package.jsonindex 9495715d..5a786aae 100644--- a/apps/extension/package.json+++ b/apps/extension/package.json@@ -9,11 +9,12 @@"dev": "vite --config vite.config.ts","lint:firefox": "web-ext lint --source-dir dist-firefox --output text","package:firefox": "bun run build:firefox && bun run lint:firefox && web-ext build --source-dir dist-firefox --artifacts-dir web-ext-artifacts --overwrite-dest",- "test": "bun test manifest.test.mjs context-menus.test.mjs popup.test.mjs capture-bundle.test.mjs image-clip.test.mjs tweet-clip.test.mjs selection-clip.test.mjs github-repo.test.mjs xhs-clip.test.mjs zhihu-clip.test.mjs reddit-clip.test.mjs video-clip.test.mjs"+ "test": "bun test manifest.test.mjs context-menus.test.mjs popup.test.mjs capture-bundle.test.mjs image-clip.test.mjs tweet-clip.test.mjs selection-clip.test.mjs github-repo.test.mjs xhs-clip.test.mjs zhihu-clip.test.mjs reddit-clip.test.mjs video-clip.test.mjs page-images.test.mjs"},"dependencies": {"@mixmark-io/domino": "2.2.0","@mozilla/readability": "^0.6.0",+ "marked": "^18.0.5","turndown": "^7.2.4"},"devDependencies": {diff --git a/apps/extension/page-images.test.mjs b/apps/extension/page-images.test.mjsnew file mode 100644index 00000000..828a5e44--- /dev/null+++ b/apps/extension/page-images.test.mjs@@ -0,0 +1,294 @@+import { readFileSync } from "node:fs";+import { describe, expect, test } from "bun:test";+import {+ embedPageImages,+ MAX_PAGE_IMAGE_TOTAL_BYTES,+ MAX_PAGE_IMAGES,+ pageImageRefs,+ readBodyWithLimit,+ replacePageImageUrls,+} from "./src/page-images.ts";++const png = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]);+const created = { revision: 1, contentHash: "hash" };+const pngFile = { bytes: png, mimeType: "image/png" };++const createClient = ({ failUploadFor = [], failSave = false, session = created, upload } = {}) => {+ const calls = { uploads: [], saves: [], sessions: 0 };+ let next = 0;+ return {+ calls,+ client: {+ uploadImage: async (memoId, file, signal) => {+ if (upload) return upload(memoId, file, signal);+ if (failUploadFor.includes(file.filename)) throw new Error("upload failed");+ calls.uploads.push({ memoId, file });+ next += 1;+ return { id: `res_${next}` };+ },+ createEditSession: async () => {+ calls.sessions += 1;+ return { editSession: { id: "es_1", baseRevision: session.revision, baseContentHash: session.contentHash } };+ },+ saveMemo: async (memoId, body) => {+ if (failSave) throw new Error("conflict");+ calls.saves.push({ memoId, body });+ return {};+ },+ },+ };+};++const onlyFrom = (urls) => async (image) => (urls.includes(image.url) ? pngFile : null);++describe("page clip images", () => {+ test("finds remote image addresses, resolving relative ones against the page", () => {+ const markdown = [+ "",+ "",+ "",+ "",+ "",+ "[not an image](https://example.com/page)",+ "",+ ].join("\n\n");++ expect(pageImageRefs(markdown, "https://example.com/post/1")).toEqual([+ { href: "https://cdn.example.com/hero.png", url: "https://cdn.example.com/hero.png" },+ { href: "/img/a.jpg", url: "https://example.com/img/a.jpg" },+ { href: "https://cdn.example.com/with space.png", url: "https://cdn.example.com/with%20space.png" },+ ]);+ });++ test("ignores image syntax inside code blocks and code spans", () => {+ const markdown = [+ "Inline `` example.",+ " ",+ "```md\n\n```",+ "- item\n\n ```\n \n ```\n- ",+ ].join("\n\n");++ expect(pageImageRefs(markdown, "https://example.com")).toEqual([+ { href: "https://cdn.example.com/a.png", url: "https://cdn.example.com/a.png" },+ ]);+ expect(replacePageImageUrls(markdown, [{ href: "https://cdn.example.com/a.png", resourceId: "res_1" }])).toBe(+ markdown.replace("", ""),+ );+ });++ test("reads destinations with escaped and nested parentheses", () => {+ const markdown = [+ ".png)",+ "![B [x]](https://example.com/p(1).png \"Say \\\"hi\\\"\")",+ "> quoted ",+ ].join("\n\n");++ expect(pageImageRefs(markdown, "https://example.com")).toEqual([+ { href: "https://example.com/a_(b).png", url: "https://example.com/a_(b).png" },+ { href: "https://example.com/p(1).png", url: "https://example.com/p(1).png" },+ { href: "https://example.com/c.png", url: "https://example.com/c.png" },+ ]);+ expect(replacePageImageUrls(markdown, [+ { href: "https://example.com/a_(b).png", resourceId: "res_1" },+ { href: "https://example.com/p(1).png", resourceId: "res_2" },+ { href: "https://example.com/c.png", resourceId: "res_3" },+ ])).toBe([+ "",+ "![B [x]](/api/v1/resources/res_2/blob \"Say \\\"hi\\\"\")",+ "> quoted ",+ ].join("\n\n"));+ });++ test("caps how many images one clip copies", () => {+ const markdown = Array.from({ length: MAX_PAGE_IMAGES + 5 }, (_, index) => ``).join("\n");+ expect(pageImageRefs(markdown, "https://example.com")).toHaveLength(MAX_PAGE_IMAGES);+ });++ test("rewrites only uploaded images and keeps alt text and titles", () => {+ const markdown = "\n\n";+ expect(replacePageImageUrls(markdown, [{ href: "https://cdn.example.com/hero.png", resourceId: "res_9" }])).toBe(+ "\n\n",+ );+ });++ test("uploads downloaded images and saves the rewritten note once", async () => {+ const markdown = "# Post\n\n\n\n";+ const { client, calls } = createClient();++ const result = await embedPageImages(client, {+ memoId: "memo_1",+ markdown,+ created,+ images: pageImageRefs(markdown, "https://example.com"),+ download: onlyFrom(["https://cdn.example.com/a.png"]),+ });++ expect(result).toEqual({ embedded: 1, total: 2 });+ expect(calls.uploads).toHaveLength(1);+ expect(calls.saves).toEqual([{+ memoId: "memo_1",+ body: {+ editSessionId: "es_1",+ expectedRevision: 1,+ expectedContentHash: "hash",+ contentMarkdown: "# Post\n\n\n\n",+ },+ }]);+ });++ test("does not overwrite a note edited while its images were copied", async () => {+ const markdown = "";+ for (const session of [{ revision: 2, contentHash: "hash" }, { revision: 1, contentHash: "edited" }]) {+ const { client, calls } = createClient({ session });+ const result = await embedPageImages(client, {+ memoId: "memo_1",+ markdown,+ created,+ images: pageImageRefs(markdown, "https://example.com"),+ download: onlyFrom(["https://cdn.example.com/a.png"]),+ });+ expect(result).toEqual({ embedded: 0, total: 1 });+ expect(calls.sessions).toBe(1);+ expect(calls.saves).toHaveLength(0);+ }+ });++ test("leaves the saved note untouched when nothing could be copied", async () => {+ const markdown = "";+ const { client, calls } = createClient({ failUploadFor: ["a.png"] });++ const result = await embedPageImages(client, {+ memoId: "memo_1",+ markdown,+ created,+ images: pageImageRefs(markdown, "https://example.com"),+ download: onlyFrom(["https://cdn.example.com/a.png"]),+ });++ expect(result).toEqual({ embedded: 0, total: 1 });+ expect(calls.sessions).toBe(0);+ expect(calls.saves).toHaveLength(0);+ });++ test("does not throw when the rewrite conflicts", async () => {+ const markdown = "";+ const { client } = createClient({ failSave: true });+ const result = await embedPageImages(client, {+ memoId: "memo_1",+ markdown,+ created,+ images: pageImageRefs(markdown, "https://example.com"),+ download: onlyFrom(["https://cdn.example.com/a.png"]),+ });+ expect(result).toEqual({ embedded: 0, total: 1 });+ });++ test("holds one image at a time and shrinks the byte limit as images upload", async () => {+ const markdown = Array.from({ length: 5 }, (_, index) => ``).join("\n\n");+ const { client } = createClient();+ const events = [];+ const limits = [];+ const size = 15 * 1024 * 1024;+ const result = await embedPageImages({+ ...client,+ uploadImage: async (memoId, file, signal) => {+ events.push(`upload ${file.filename}`);+ return client.uploadImage(memoId, file, signal);+ },+ }, {+ memoId: "memo_1",+ markdown,+ created,+ images: pageImageRefs(markdown, "https://example.com"),+ download: async (image, { maxBytes }) => {+ events.push(`download ${image.url.split("/").at(-1)}`);+ limits.push(maxBytes);+ const bytes = new Uint8Array(Math.min(size, maxBytes));+ bytes.set(png);+ return { bytes, mimeType: "image/png" };+ },+ });++ expect(events.slice(0, 4)).toEqual(["download 0.png", "upload 0.png", "download 1.png", "upload 1.png"]);+ expect(limits[0]).toBe(20 * 1024 * 1024);+ expect(limits.at(-1)).toBe(MAX_PAGE_IMAGE_TOTAL_BYTES - 3 * size);+ expect(limits).toHaveLength(4);+ expect(result.embedded).toBe(4);+ });++ test("stops waiting on stalled downloads and uploads once the budget is spent", async () => {+ const markdown = "\n\n\n\n";+ const never = new Promise(() => {});+ const signals = [];+ const { client, calls } = createClient({+ upload: (_memoId, file, signal) => {+ signals.push(signal);+ return file.filename === "b.png" ? never : Promise.resolve({ id: "res_a" });+ },+ });++ const startedAt = Date.now();+ const result = await embedPageImages(client, {+ memoId: "memo_1",+ markdown,+ created,+ images: pageImageRefs(markdown, "https://example.com"),+ download: async (image, { timeoutMs }) => {+ expect(timeoutMs).toBeLessThanOrEqual(100);+ return image.url.endsWith("c.png") ? never : pngFile;+ },+ transferBudgetMs: 100,+ budgetMs: 200,+ });++ expect(Date.now() - startedAt).toBeLessThan(1000);+ expect(signals.at(-1)?.aborted).toBe(true);+ expect(result).toEqual({ embedded: 1, total: 3 });+ expect(calls.saves[0].body.contentMarkdown).toBe(+ "\n\n\n\n",+ );+ });++ test("gives up on the rewrite when the server does not answer in time", async () => {+ const markdown = "";+ const { client } = createClient();+ const startedAt = Date.now();+ const result = await embedPageImages({ ...client, createEditSession: () => new Promise(() => {}) }, {+ memoId: "memo_1",+ markdown,+ created,+ images: pageImageRefs(markdown, "https://example.com"),+ download: onlyFrom(["https://cdn.example.com/a.png"]),+ transferBudgetMs: 50,+ budgetMs: 100,+ });+ expect(Date.now() - startedAt).toBeLessThan(1000);+ expect(result).toEqual({ embedded: 0, total: 1 });+ });++ test("stops reading a body as soon as it exceeds the limit", async () => {+ let pulled = 0;+ const stream = new ReadableStream({+ pull(controller) {+ pulled += 1;+ controller.enqueue(new Uint8Array(1024));+ if (pulled > 100) controller.close();+ },+ });+ expect(await readBodyWithLimit(new Response(stream), 4096)).toBeNull();+ expect(pulled).toBeLessThan(10);+ expect(await readBodyWithLimit(new Response(new Uint8Array(10)), 4096)).toHaveLength(10);+ expect(await readBodyWithLimit(+ new Response(new Uint8Array(10), { headers: { "content-length": "9999" } }),+ 4096,+ )).toBeNull();+ });++ test("page and selection clips copy their images after the note is created", () => {+ const background = readFileSync(new URL("./src/background.ts", import.meta.url), "utf8");+ expect(background).toContain("await createMemo(settings, page, tab.id);");+ expect(background).toContain("await embedClipImages(settings, created.memo, contentMarkdown, page.url, tabId, null);");+ expect(background).toContain("await embedClipImages(settings, created.memo, contentMarkdown, sourceUrl, tabId, frameId);");+ expect(background).toContain("func: readPageImageInPage");+ });+});diff --git a/apps/extension/src/background.ts b/apps/extension/src/background.tsindex 0a079f0b..0e462e6b 100644--- a/apps/extension/src/background.ts+++ b/apps/extension/src/background.ts@@ -24,6 +24,13 @@ import {type StoredImage,type StoredImageFailure,} from "./image-clip";+import {+ embedPageImages,+ pageImageRefs,+ readBodyWithLimit,+ readPageImageInPage,+ type PageImageDownload,+} from "./page-images";import {githubRepoFactsFromSource,githubRepoNoteMarkdown,@@ -186,16 +193,18 @@ const notebookForClip = async (settings: ExtensionSettings) => {return notebookId;};-const createMemo = async (settings: ExtensionSettings, page: CapturedPage) => {- await edgeEverRequest(settings, "/api/v1/memos", {+const createMemo = async (settings: ExtensionSettings, page: CapturedPage, tabId: number | null) => {+ const contentMarkdown = toMarkdown(page);+ const created = await edgeEverRequest<{ memo?: CreatedClipMemo }>(settings, "/api/v1/memos", {method: "POST",body: JSON.stringify({notebookId: await notebookForClip(settings),title: page.title,- contentMarkdown: toMarkdown(page),+ contentMarkdown,tags: ["web-clip"],}),});+ await embedClipImages(settings, created.memo, contentMarkdown, page.url, tabId, null);};const createGithubRepoMemo = async (settings: ExtensionSettings, facts: GithubRepoFacts) => {@@ -230,8 +239,8 @@ const imageNoteClient = (settings: ExtensionSettings): ImageNoteClient => ({method: "POST",body: JSON.stringify(body),}),- uploadImage: async (memoId, file) => {- const uploaded = await uploadMemoImage(settings, memoId, file);+ uploadImage: async (memoId, file, signal) => {+ const uploaded = await uploadMemoImage(settings, memoId, file, signal);return uploaded.resource;},createEditSession: (memoId) => edgeEverRequest(@@ -367,21 +376,36 @@ const persistImage = async (}};-const downloadImage = async (urls: string[]): Promise<StoredImage | StoredImageFailure> => {+const requestSignal = (timeoutMs: number, signal?: AbortSignal) => {+ if (!signal) return AbortSignal.timeout(timeoutMs);+ const controller = new AbortController();+ const abort = () => controller.abort();+ const timer = setTimeout(abort, timeoutMs);+ controller.signal.addEventListener("abort", () => clearTimeout(timer), { once: true });+ if (signal.aborted) abort();+ else signal.addEventListener("abort", abort, { once: true });+ return controller.signal;+};++const downloadImage = async (+ urls: string[],+ { maxBytes = MAX_IMAGE_BYTES, timeoutMs = 15000, signal }: { maxBytes?: number; timeoutMs?: number; signal?: AbortSignal } = {},+): Promise<StoredImage | StoredImageFailure> => {const matchUrl = urls[urls.length - 1] ?? "";let matchError: StoredImageFailure["error"] | "" = "";for (const url of urls) {if (!/^https?:/i.test(url)) continue;for (const credentials of ["omit", "include"] as const) {+ if (signal?.aborted) return { error: "unreadable" };try {- const response = await fetch(url, { credentials });+ const response = await fetch(url, { credentials, signal: requestSignal(timeoutMs, signal) });if (!response.ok) continue;- const declared = Number(response.headers.get("content-length"));- if (Number.isFinite(declared) && declared > MAX_IMAGE_BYTES) {+ const bytes = await readBodyWithLimit(response, maxBytes);+ if (!bytes) {if (url === matchUrl) matchError = "too-large";continue;}- const image = imageFromBytes(new Uint8Array(await response.arrayBuffer()), response.headers.get("content-type") ?? "");+ const image = imageFromBytes(bytes, response.headers.get("content-type") ?? "");if (!("error" in image)) return image;if (url === matchUrl) matchError = image.error;} catch {@@ -393,6 +417,56 @@ const downloadImage = async (urls: string[]): Promise<StoredImage | StoredImageFreturn { error: "unreadable" };};+type CreatedClipMemo = { id?: string; revision?: number; contentHash?: string };++const downloadPageImage = (tabId: number | null, frameId: number | null): PageImageDownload =>+ async (image, { maxBytes, timeoutMs, signal }) => {+ const file = await downloadImage([image.url], { maxBytes, timeoutMs, signal });+ if (!("error" in file)) return file;+ if (tabId === null || signal.aborted) return null;+ try {+ const [injection] = await chrome.scripting.executeScript({+ target: scriptTarget(tabId, frameId),+ func: readPageImageInPage,+ args: [image.url, maxBytes, timeoutMs],+ });+ const read: unknown = injection?.result;+ if (!read || typeof read !== "object") return null;+ const { base64, type } = read as { base64?: unknown; type?: unknown };+ if (typeof base64 !== "string") return null;+ const fromPage = imageFromBase64(base64, typeof type === "string" ? type : "");+ return "error" in fromPage ? null : fromPage;+ } catch {+ // The page cannot be scripted; this image keeps its remote address.+ return null;+ }+ };++const embedClipImages = async (+ settings: ExtensionSettings,+ memo: CreatedClipMemo | undefined,+ markdown: string,+ pageUrl: string,+ tabId: number | null,+ frameId: number | null,+) => {+ if (!memo?.id || typeof memo.revision !== "number" || typeof memo.contentHash !== "string") return;+ const images = pageImageRefs(markdown, pageUrl);+ if (images.length === 0) return;+ try {+ await embedPageImages(imageNoteClient(settings), {+ memoId: memo.id,+ markdown,+ created: { revision: memo.revision, contentHash: memo.contentHash },+ images,+ download: downloadPageImage(tabId, frameId),+ });+ } catch {+ // The note is already saved; reporting a failure here would invite a+ // duplicate save. Its images keep their remote addresses.+ }+};+const hasImagePermission = async (urls: string[]) => {const patterns = [...new Set(urls.map(imageOriginPattern).filter((pattern): pattern is string => Boolean(pattern)))];for (const pattern of patterns) {@@ -1270,21 +1344,24 @@ const saveSelectionFromMenu = async (const markdown = fromSelection?.markdown.trim() || browserSelection;const plain = fromSelection?.plainText?.trim() || browserSelection || markdown;if (!markdown.trim()) throw new Error(t("selectionEmpty"));- await edgeEverRequest(settings, "/api/v1/memos", {+ const sourceUrl = fromSelection?.url || pageUrl;+ const contentMarkdown = selectionNoteMarkdown({+ markdown,+ pageUrl: sourceUrl,+ capturedAt: new Date().toISOString(),+ sourceLabel: t("sourceLabel"),+ capturedAtLabel: t("capturedAtLabel"),+ });+ const created = await edgeEverRequest<{ memo?: CreatedClipMemo }>(settings, "/api/v1/memos", {method: "POST",body: JSON.stringify({notebookId: await notebookForClip(settings),title: selectionNoteTitle(plain, pageTitle, t("selectionNoteFallbackTitle")),- contentMarkdown: selectionNoteMarkdown({- markdown,- pageUrl: fromSelection?.url || pageUrl,- capturedAt: new Date().toISOString(),- sourceLabel: t("sourceLabel"),- capturedAtLabel: t("capturedAtLabel"),- }),+ contentMarkdown,tags: ["web-clip"],}),});+ await embedClipImages(settings, created.memo, contentMarkdown, sourceUrl, tabId, frameId);await showFeedback(tabId, frameId, t("selectionSaved"), "success");} catch (error) {const message = describeSelectionError(error);@@ -1752,7 +1829,7 @@ chrome.runtime.onMessage.addListener((message: { type?: string; page?: CapturedP}const page = await readCapturedPage(tab.id, null, "page");- await createMemo(settings, page);+ await createMemo(settings, page, tab.id);sendResponse({ ok: true });} catch (error) {const message = error instanceof Error ? error.message : "";diff --git a/apps/extension/src/extension.ts b/apps/extension/src/extension.tsindex 914f6a9e..27bcbb0e 100644--- a/apps/extension/src/extension.ts+++ b/apps/extension/src/extension.ts@@ -76,7 +76,12 @@ export const edgeEverRequest = async <T>(settings: ExtensionSettings, path: strireturn response.json() as Promise<T>;};-export const edgeEverFormRequest = async <T>(settings: ExtensionSettings, path: string, form: FormData): Promise<T> => {+export const edgeEverFormRequest = async <T>(+ settings: ExtensionSettings,+ path: string,+ form: FormData,+ signal?: AbortSignal,+): Promise<T> => {const instanceUrl = normalizeInstanceUrl(settings.instanceUrl);if (!instanceUrl || !settings.token) {throw new Error(t("missingSettingsDetails"));@@ -86,6 +91,7 @@ export const edgeEverFormRequest = async <T>(settings: ExtensionSettings, path:method: "POST",headers: { Authorization: `Bearer ${settings.token}` },body: form,+ signal,});if (!response.ok) {@@ -100,6 +106,7 @@ export const uploadMemoImage = async (settings: ExtensionSettings,memoId: string,file: { bytes: Uint8Array; mimeType: string; filename: string },+ signal?: AbortSignal,) => {const buffer = new ArrayBuffer(file.bytes.byteLength);new Uint8Array(buffer).set(file.bytes);@@ -109,6 +116,7 @@ export const uploadMemoImage = async (settings,`/api/v1/memos/${encodeURIComponent(memoId)}/resources`,form,+ signal,);};diff --git a/apps/extension/src/image-clip.ts b/apps/extension/src/image-clip.tsindex de3887e0..6f601d70 100644--- a/apps/extension/src/image-clip.ts+++ b/apps/extension/src/image-clip.ts@@ -266,6 +266,7 @@ export type ImageNoteClient = {uploadImage: (memoId: string,file: { bytes: Uint8Array; mimeType: string; filename: string },+ signal?: AbortSignal,) => Promise<{ id: string }>;createEditSession: (memoId: string) => Promise<{editSession: { id: string; baseRevision: number; baseContentHash: string };diff --git a/apps/extension/src/page-images.ts b/apps/extension/src/page-images.tsnew file mode 100644index 00000000..8273a56f--- /dev/null+++ b/apps/extension/src/page-images.ts@@ -0,0 +1,292 @@+import { Lexer, marked, type Token } from "marked";+import { filenameForImage, MAX_IMAGE_BYTES, type ImageNoteClient, type StoredImage } from "./image-clip";++// A long article can reference hundreds of images. Keep the extra uploads after+// a clip bounded; the remaining images keep their original addresses.+export const MAX_PAGE_IMAGES = 30;+export const MAX_PAGE_IMAGE_TOTAL_BYTES = 60 * 1024 * 1024;+// The whole copy, from the first download to the rewritten note, finishes+// within this budget so the popup or selection feedback never waits longer.+export const PAGE_IMAGE_BUDGET_MS = 45_000;+// No new download or upload starts after this point; the rest of the budget+// is kept for rewriting the note.+export const PAGE_IMAGE_TRANSFER_BUDGET_MS = 35_000;+export const PAGE_IMAGE_REQUEST_TIMEOUT_MS = 15_000;++export type PageImageRef = {+ /** The image destination as Markdown parses it (escapes resolved). */+ href: string;+ /** The absolute http(s) address used to download the image. */+ url: string;+};++type LocatedImage = {+ href: string;+ title: string | null;+ /** Offset just past the closing `]` of the image label. */+ labelEnd: number;+ end: number;+};++// Code blocks, code spans and raw HTML can contain text that looks like an+// image; they are located only so the search below steps over them.+const STEPPED_OVER_TOKENS = new Set(["code", "codespan", "html"]);++const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");++// Tokens nested in list items and block quotes are lexed without their+// indentation or `>` markers, so their raw text may not appear verbatim.+const findRaw = (markdown: string, raw: string, from: number) => {+ const exact = markdown.indexOf(raw, from);+ if (exact !== -1) return { start: exact, end: exact + raw.length };+ const pattern = new RegExp(raw.split("\n").map(escapeRegExp).join("\\n[ \\t>]*"), "g");+ pattern.lastIndex = from;+ const match = pattern.exec(markdown);+ return match ? { start: match.index, end: match.index + match[0].length } : null;+};++const imageLabelEnd = (source: string) => {+ let depth = 0;+ for (let index = 1; index < source.length; index += 1) {+ const char = source[index];+ if (char === "\\") {+ index += 1;+ } else if (char === "[") {+ depth += 1;+ } else if (char === "]") {+ depth -= 1;+ if (depth === 0) return index + 1;+ }+ }+ return -1;+};++const locatePageImages = (markdown: string): LocatedImage[] => {+ const located: LocatedImage[] = [];+ let cursor = 0;+ let lost = false;+ marked.walkTokens(new Lexer().lex(markdown), (token: Token) => {+ if (lost || (token.type !== "image" && !STEPPED_OVER_TOKENS.has(token.type))) return;+ const range = findRaw(markdown, token.raw, cursor);+ if (!range) {+ // Stop rather than guess: images after this point keep their addresses.+ lost = true;+ return;+ }+ cursor = range.end;+ if (token.type !== "image") return;+ const labelEnd = imageLabelEnd(markdown.slice(range.start, range.end));+ if (labelEnd < 0) return;+ located.push({+ href: token.href,+ title: token.title ?? null,+ labelEnd: range.start + labelEnd,+ end: range.end,+ });+ });+ return located;+};++const absoluteImageUrl = (href: string, baseUrl: string) => {+ try {+ const url = new URL(href, baseUrl || undefined);+ return url.protocol === "http:" || url.protocol === "https:" ? url.href : null;+ } catch {+ return null;+ }+};++export const pageImageRefs = (markdown: string, baseUrl: string, limit = MAX_PAGE_IMAGES): PageImageRef[] => {+ const refs: PageImageRef[] = [];+ const seen = new Set<string>();+ for (const { href } of locatePageImages(markdown)) {+ if (!href || seen.has(href) || href.startsWith("/api/v1/resources/")) continue;+ const url = absoluteImageUrl(href, baseUrl);+ if (!url) continue;+ seen.add(href);+ refs.push({ href, url });+ if (refs.length >= limit) break;+ }+ return refs;+};++export const replacePageImageUrls = (+ markdown: string,+ uploaded: ReadonlyArray<{ href: string; resourceId: string }>,+) => {+ const resources = new Map(uploaded.map((image) => [image.href, image.resourceId]));+ if (resources.size === 0) return markdown;+ let result = markdown;+ for (const image of locatePageImages(markdown).reverse()) {+ const resourceId = resources.get(image.href);+ if (!resourceId) continue;+ const title = image.title ? ` "${image.title.replace(/["\\]/g, "\\$&")}"` : "";+ result = `${result.slice(0, image.labelEnd)}(/api/v1/resources/${encodeURIComponent(resourceId)}/blob${title})${result.slice(image.end)}`;+ }+ return result;+};++/** Settles with `work`, or rejects as soon as `signal` aborts. */+const untilAborted = <T>(work: Promise<T>, signal: AbortSignal) =>+ new Promise<T>((resolve, reject) => {+ if (signal.aborted) {+ reject(signal.reason);+ return;+ }+ const onAbort = () => reject(signal.reason);+ signal.addEventListener("abort", onAbort, { once: true });+ work.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));+ });++/** Reads a response body, giving up as soon as it exceeds `maxBytes`. */+export const readBodyWithLimit = async (response: Response, maxBytes: number) => {+ const declared = Number(response.headers.get("content-length"));+ if (Number.isFinite(declared) && declared > maxBytes) return null;+ if (!response.body) {+ const bytes = new Uint8Array(await response.arrayBuffer());+ return bytes.byteLength > maxBytes ? null : bytes;+ }+ const reader = response.body.getReader();+ const chunks: Uint8Array[] = [];+ let size = 0;+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ size += value.byteLength;+ if (size > maxBytes) {+ await reader.cancel().catch(() => undefined);+ return null;+ }+ chunks.push(value);+ }+ const bytes = new Uint8Array(size);+ let offset = 0;+ for (const chunk of chunks) {+ bytes.set(chunk, offset);+ offset += chunk.byteLength;+ }+ return bytes;+};++export type PageImageDownload = (+ image: PageImageRef,+ limits: { maxBytes: number; timeoutMs: number; signal: AbortSignal },+) => Promise<StoredImage | null>;++/**+ * Copies a saved clip's remote images into the note's own resources, then+ * rewrites the note to point at them. The note is already saved before this+ * runs: any image that cannot be downloaded or uploaded keeps its original+ * address, and a failed rewrite leaves the first version untouched.+ *+ * Images are downloaded and uploaded one at a time so at most one is held in+ * memory, and the byte and time limits cover the whole operation.+ */+export const embedPageImages = async (+ client: Pick<ImageNoteClient, "uploadImage" | "createEditSession" | "saveMemo">,+ input: {+ memoId: string;+ markdown: string;+ /** The revision and content hash returned when the note was created. */+ created: { revision: number; contentHash: string };+ images: readonly PageImageRef[];+ download: PageImageDownload;+ budgetMs?: number;+ transferBudgetMs?: number;+ },+) => {+ const total = input.images.length;+ if (total === 0) return { embedded: 0, total };+ const startedAt = Date.now();+ const transferDeadline = startedAt + (input.transferBudgetMs ?? PAGE_IMAGE_TRANSFER_BUDGET_MS);+ const transfers = AbortSignal.timeout(input.transferBudgetMs ?? PAGE_IMAGE_TRANSFER_BUDGET_MS);+ const workflow = AbortSignal.timeout(input.budgetMs ?? PAGE_IMAGE_BUDGET_MS);++ const uploaded: Array<{ href: string; resourceId: string }> = [];+ let remainingBytes = MAX_PAGE_IMAGE_TOTAL_BYTES;+ for (const image of input.images) {+ if (transfers.aborted || remainingBytes <= 0) break;+ try {+ const maxBytes = Math.min(MAX_IMAGE_BYTES, remainingBytes);+ const timeoutMs = Math.min(PAGE_IMAGE_REQUEST_TIMEOUT_MS, transferDeadline - Date.now());+ const file = await untilAborted(input.download(image, { maxBytes, timeoutMs, signal: transfers }), transfers);+ if (!file || file.bytes.byteLength > maxBytes) continue;+ const resource = await untilAborted(client.uploadImage(input.memoId, {+ bytes: file.bytes,+ mimeType: file.mimeType,+ filename: filenameForImage(image.url, file.mimeType),+ }, transfers), transfers);+ remainingBytes -= file.bytes.byteLength;+ if (resource.id) uploaded.push({ href: image.href, resourceId: resource.id });+ } catch {+ // Keep the remote address for this image.+ }+ }+ if (uploaded.length === 0) return { embedded: 0, total };++ try {+ const { editSession } = await untilAborted(client.createEditSession(input.memoId), workflow);+ // Someone edited the note while its images were copied: keep their+ // version instead of writing the clip over it.+ if (editSession.baseRevision !== input.created.revision+ || editSession.baseContentHash !== input.created.contentHash) {+ return { embedded: 0, total };+ }+ await untilAborted(client.saveMemo(input.memoId, {+ editSessionId: editSession.id,+ expectedRevision: editSession.baseRevision,+ expectedContentHash: editSession.baseContentHash,+ contentMarkdown: replacePageImageUrls(input.markdown, uploaded),+ }), workflow);+ } catch {+ return { embedded: 0, total };+ }+ return { embedded: uploaded.length, total };+};++// Runs in the clipped page so same-origin and CORS-enabled images can be read+// with the page's own session when the extension itself cannot fetch them.+// Keep this function free of closures: Chrome serializes it with toString().+export async function readPageImageInPage(+ url: string,+ maxBytes: number,+ timeoutMs: number,+): Promise<{ base64: string; type: string } | null> {+ const deadline = Date.now() + timeoutMs;+ for (const credentials of ["include", "omit"] as const) {+ const remaining = deadline - Date.now();+ if (remaining <= 0) return null;+ try {+ const response = await fetch(url, { credentials, signal: AbortSignal.timeout(remaining) });+ if (!response.ok) continue;+ const declared = Number(response.headers.get("content-length"));+ if (Number.isFinite(declared) && declared > maxBytes) return null;+ if (!response.body) return null;+ const reader = response.body.getReader();+ const chunks: Uint8Array[] = [];+ let size = 0;+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ size += value.byteLength;+ if (size > maxBytes) {+ await reader.cancel().catch(() => undefined);+ return null;+ }+ chunks.push(value);+ }+ if (size === 0) return null;+ const blob = new Blob(chunks as BlobPart[]);+ const base64 = await new Promise<string>((resolve, reject) => {+ const fileReader = new FileReader();+ fileReader.onload = () => resolve(String(fileReader.result).replace(/^data:[^,]*,/, ""));+ fileReader.onerror = () => reject(fileReader.error);+ fileReader.readAsDataURL(blob);+ });+ return { base64, type: response.headers.get("content-type") ?? "" };+ } catch {+ // Try the next credentials mode, then give up on this image.+ }+ }+ return null;+}diff --git a/bun.lock b/bun.lockindex 9bf04e85..1a0fbe66 100644--- a/bun.lock+++ b/bun.lock@@ -123,6 +123,7 @@"dependencies": {"@mixmark-io/domino": "2.2.0","@mozilla/readability": "^0.6.0",+ "marked": "^18.0.5","turndown": "^7.2.4",},"devDependencies": {diff --git a/tests/e2e/web-clipper-page-images.spec.ts b/tests/e2e/web-clipper-page-images.spec.tsnew file mode 100644index 00000000..a7c544a8--- /dev/null+++ b/tests/e2e/web-clipper-page-images.spec.ts@@ -0,0 +1,181 @@+import { chromium, expect, test, type BrowserContext, type Worker } from "@playwright/test";+import { execFileSync } from "node:child_process";+import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { createServer, type Server } from "node:http";+import type { AddressInfo } from "node:net";+import { tmpdir } from "node:os";+import { join, resolve } from "node:path";+import { fileURLToPath } from "node:url";++// Extension APIs, available inside the evaluated extension contexts.+declare const chrome: any;++// Loads the built Web Clipper into Chromium, saves a real page through the+// popup's `captureCurrentPage` message and checks that its images were copied+// into the note.+const repositoryRoot = resolve(fileURLToPath(new URL("../..", import.meta.url)));+const png = Buffer.from(+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",+ "base64",+);+const filler = "This paragraph gives the reader view enough article text to keep the content block. ".repeat(12);++type ImageRequest = { path: string; status: number; fromPage: boolean; script: boolean };++const startSourceSite = async (title: string) => {+ const requests: ImageRequest[] = [];+ const server = createServer((request, response) => {+ const { pathname } = new URL(request.url ?? "/", "http://127.0.0.1");+ const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+ const fromPage = (request.headers.referer ?? "").startsWith(`${origin}/`);+ // fetch() rather than an <img> element loading the picture.+ const script = request.headers["sec-fetch-dest"] === "empty";+ const send = (status: number, type: string, body: string | Buffer) => {+ if (pathname.endsWith(".png")) requests.push({ path: pathname, status, fromPage, script });+ response.writeHead(status, { "content-type": type });+ response.end(body);+ };+ if (pathname === "/article.html") {+ send(200, "text/html; charset=utf-8", `<!doctype html>+<html><head><title>${title}</title></head><body>+<article>+ <h1>${title}</h1>+ <p>${filler}</p>+ <p><img src="/open.png" alt="Open image"></p>+ <p>${filler}</p>+ <p><img src="/protected.png" alt="Hotlink protected image"></p>+ <p>${filler}</p>+ <p><img src="/missing.png" alt="Missing image"></p>+ <pre><code></code></pre>+ <p>${filler}</p>+</article>+</body></html>`);+ return;+ }+ if (pathname === "/open.png" || pathname === "/code.png") {+ send(200, "image/png", png);+ return;+ }+ // Hotlink protection: only requests sent by the page itself carry its+ // address as the referrer, so the extension must read this one in-page.+ if (pathname === "/protected.png") {+ if (fromPage) send(200, "image/png", png);+ else send(403, "text/plain", "forbidden");+ return;+ }+ send(404, "text/plain", "not found");+ });+ await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));+ const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+ return { server, origin, requests };+};++const prepareExtension = () => {+ execFileSync("bun", ["run", "build:extension"], { cwd: repositoryRoot, stdio: "pipe" });+ const directory = mkdtempSync(join(tmpdir(), "edgeever-web-clipper-"));+ cpSync(join(repositoryRoot, "apps/extension/dist"), directory, { recursive: true });+ // Automation cannot click the toolbar button (activeTab) or accept the+ // instance permission prompt, so this copy is granted the local hosts the+ // user would otherwise grant.+ const manifestPath = join(directory, "manifest.json");+ const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));+ manifest.host_permissions = ["http://127.0.0.1/*"];+ writeFileSync(manifestPath, JSON.stringify(manifest));+ return directory;+};++test.describe("web clipper page images", () => {+ let extensionDirectory = "";+ let userDataDirectory = "";+ let context: BrowserContext | null = null;+ let site: Awaited<ReturnType<typeof startSourceSite>> | null = null;++ test.afterEach(async () => {+ await context?.close();+ await new Promise((done) => (site?.server as Server | undefined)?.close(done) ?? done(undefined));+ if (extensionDirectory) rmSync(extensionDirectory, { recursive: true, force: true });+ if (userDataDirectory) rmSync(userDataDirectory, { recursive: true, force: true });+ });++ test("copies readable images into the saved note and leaves the rest untouched", async ({ request, baseURL }) => {+ test.setTimeout(180_000);+ const title = `Web clipper image check ${Date.now()}`;+ site = await startSourceSite(title);+ extensionDirectory = prepareExtension();+ userDataDirectory = mkdtempSync(join(tmpdir(), "edgeever-web-clipper-profile-"));++ const tokenResponse = await request.post("/api/v1/api-tokens", {+ data: { name: "web-clipper-e2e", scopes: ["read:memos", "write:memos", "read:resources", "write:resources", "read:notebooks"] },+ });+ expect(tokenResponse.status()).toBe(201);+ const { token, apiToken } = await tokenResponse.json() as { token: string; apiToken: { id: string } };++ let memoId = "";+ try {+ context = await chromium.launchPersistentContext(userDataDirectory, {+ channel: "chromium",+ args: [`--disable-extensions-except=${extensionDirectory}`, `--load-extension=${extensionDirectory}`],+ });+ const worker: Worker = context.serviceWorkers()[0] ?? await context.waitForEvent("serviceworker");+ await worker.evaluate((settings) => chrome.storage.local.set(settings), {+ instanceUrl: baseURL ?? "",+ token,+ notebookId: "",+ });++ const article = context.pages()[0] ?? await context.newPage();+ await article.goto(`${site.origin}/article.html`);++ // The popup asks the background to save the active tab of the focused+ // window, so send the message from an extension page in another window.+ const senderPromise = context.waitForEvent("page");+ await worker.evaluate(() => chrome.windows.create({ url: chrome.runtime.getURL("options.html"), focused: false }));+ const sender = await senderPromise;+ await sender.waitForLoadState();+ const activeUrl = await worker.evaluate(async (articleUrl) => {+ const [tab] = await chrome.tabs.query({ url: articleUrl });+ if (tab?.windowId !== undefined) await chrome.windows.update(tab.windowId, { focused: true });+ const [active] = await chrome.tabs.query({ active: true, currentWindow: true });+ return active?.url ?? "";+ }, `${site.origin}/article.html`);+ expect(activeUrl).toBe(`${site.origin}/article.html`);++ const result = await sender.evaluate(() => chrome.runtime.sendMessage({ type: "captureCurrentPage" }));+ expect(result).toEqual({ ok: true });++ const list = await request.get("/api/v1/memos?limit=50");+ expect(list.ok()).toBe(true);+ const memos = (await list.json() as { memos: Array<{ id: string; title: string }> }).memos;+ memoId = memos.find((memo) => memo.title === title)?.id ?? "";+ expect(memoId).toBeTruthy();++ const detail = await request.get(`/api/v1/memos/${memoId}`);+ const markdown = (await detail.json() as { memo: { contentMarkdown: string } }).memo.contentMarkdown;++ const resourceIds = [...markdown.matchAll(/!\[[^\]]*\]\(\/api\/v1\/resources\/([^/)]+)\/blob\)/g)].map((match) => match[1]);+ expect(resourceIds).toHaveLength(2);+ expect(markdown).toContain(";+ expect(markdown).toContain(";+ // The clip keeps the page's own (relative) address for an image it could not copy.+ expect(markdown).toContain("");+ expect(markdown).toContain(``);+ for (const resourceId of resourceIds) {+ const blob = await request.get(`/api/v1/resources/${resourceId}/blob`);+ expect(Buffer.from(await blob.body()).equals(png)).toBe(true);+ }++ // The extension's own read was refused, and the page read succeeded.+ const protectedReads = site.requests.filter((entry) => entry.path === "/protected.png");+ expect(protectedReads.some((entry) => entry.script && !entry.fromPage && entry.status === 403)).toBe(true);+ expect(protectedReads.some((entry) => entry.script && entry.fromPage && entry.status === 200)).toBe(true);+ // Image syntax inside a code block is never downloaded.+ expect(site.requests.some((entry) => entry.path === "/code.png")).toBe(false);+ } finally {+ if (memoId) {+ await request.delete(`/api/v1/memos/${memoId}`);+ await request.delete(`/api/v1/memos/${memoId}?permanent=1`);+ }+ await request.delete(`/api/v1/api-tokens/${apiToken.id}`);+ }+ });+});