Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Resolve responsive picture sources before HTML paste

Daniel71d32dccda2d
diff --git a/app/src/protyle/util/paste.ts b/app/src/protyle/util/paste.ts
index 58dd25c516..8a16529617 100644
--- a/app/src/protyle/util/paste.ts
+++ b/app/src/protyle/util/paste.ts
@@ -1,6 +1,7 @@
import {Constants} from "../../constants";
import {isEncryptedBox} from "../../util/pathName";
import {preparePasteAssets} from "./pasteAssets";
+import {normalizePictureImages} from "./pictureImages";
import {escapeHtml, escapeMarkdownPlainText} from "../../util/escape";
import {getTableCellPlainText} from "./tableCellRich";
import {getTableCellRichContext} from "./tableCellRichContext";
@@ -766,6 +767,12 @@ export const paste = async (protyle: IProtyle, event: (ClipboardEvent | DragEven
siyuanHTML = siyuanHTML || clipboard.textSiyuan;
textHTML = clipboard.textHtml;
}
+ if (blockDOMSanitizer && !siyuanHTML && /<picture[\s>]/i.test(textHTML)) {
+ const template = document.createElement("template");
+ template.innerHTML = textHTML;
+ normalizePictureImages(template.content, getHTMLAssetSourceURL(textHTML));
+ textHTML = template.innerHTML;
+ }
if (blockDOMSanitizer && !siyuanHTML && !isProtyleUploadDisabled(protyle)) {
// 受限片段中的图片走附件上传,避免被后续纯文本降级丢弃。
const isImage = (name: string) => Constants.SIYUAN_ASSETS_IMAGE.includes(
@@ -895,7 +902,9 @@ export const paste = async (protyle: IProtyle, event: (ClipboardEvent | DragEven
// process word
const doc = new DOMParser().parseFromString(textHTML, "text/html");
// 在移除文档头之前解析来源,避免将网页根路径当成本地绝对路径。
- resolveHTMLAssetURLs(doc, getHTMLAssetSourceURL(textHTML, doc.querySelector("base[href]")?.getAttribute("href")));
+ const sourceURL = getHTMLAssetSourceURL(textHTML, doc.querySelector("base[href]")?.getAttribute("href"));
+ normalizePictureImages(doc, sourceURL);
+ resolveHTMLAssetURLs(doc, sourceURL);
if (doc.body && doc.body.innerHTML) {
textHTML = doc.body.innerHTML;
}
@@ -1326,6 +1335,7 @@ export const paste = async (protyle: IProtyle, event: (ClipboardEvent | DragEven
if (isHTML) {
const tempElement = document.createElement("div");
tempElement.innerHTML = textHTML;
+ normalizePictureImages(tempElement);
if (!getHostCapabilities().localFileSystem) {
removeHTMLLocalAssetPaths(collectHTMLLocalAssets(tempElement));
}
diff --git a/app/src/protyle/util/pictureImages.test.ts b/app/src/protyle/util/pictureImages.test.ts
new file mode 100644
index 0000000000..b2e72c9d68
--- /dev/null
+++ b/app/src/protyle/util/pictureImages.test.ts
@@ -0,0 +1,39 @@
+import {describe, it} from "node:test";
+import * as assert from "node:assert/strict";
+import {getPictureImageSource, normalizePictureImages} from "./pictureImages";
+
+describe("picture image sources", () => {
+ it("selects width and density candidates without splitting commas inside URLs", () => {
+ assert.equal(getPictureImageSource("https://example.com/image,w_320.png 320w, https://example.com/image,w_1280.png 1280w"),
+ "https://example.com/image,w_1280.png");
+ assert.equal(getPictureImageSource("https://example.com/large.png 2x, https://example.com/small.png 1x"),
+ "https://example.com/large.png");
+ assert.equal(getPictureImageSource("https://example.com/image.png,"), "https://example.com/image.png");
+ });
+
+ it("resolves relative web sources and rejects invalid candidates and unsafe schemes", () => {
+ assert.equal(getPictureImageSource("/images/a.png 320w, ../b.png 640w", "https://example.com/docs/page"),
+ "https://example.com/b.png");
+ assert.equal(getPictureImageSource("//example.com/a.png 1x", "https://other.example/page"),
+ "https://example.com/a.png");
+ assert.equal(getPictureImageSource("/images/a.png 320w"), "");
+ for (const value of ["", "https://example.com/a.png 0w", "https://example.com/a.png 0x",
+ "https://example.com/a.png 1x 2x", "javascript:alert(1) 2x", "file:///tmp/a.png 640w",
+ "data:image/png;base64,YQ== 1x", "https://example.com/a.png " + "9".repeat(400) + "w"]) {
+ assert.equal(getPictureImageSource(value), "", value);
+ }
+ });
+
+ it("does not turn an empty currentSrc into the source page URL", () => {
+ const values: Record<string, string> = {src: "data:image/png;base64,YQ=="};
+ const image = {
+ currentSrc: "",
+ closest: () => ({children: [] as Element[]}),
+ getAttribute: (attribute: string) => values[attribute] || null,
+ setAttribute: (attribute: string, value: string) => values[attribute] = value,
+ } as unknown as HTMLImageElement;
+ const root = {querySelectorAll: () => [image]} as unknown as ParentNode;
+ normalizePictureImages(root, "https://example.com/article");
+ assert.equal(values.src, "data:image/png;base64,YQ==");
+ });
+});
diff --git a/app/src/protyle/util/pictureImages.ts b/app/src/protyle/util/pictureImages.ts
new file mode 100644
index 0000000000..0757426ad0
--- /dev/null
+++ b/app/src/protyle/util/pictureImages.ts
@@ -0,0 +1,90 @@
+const resolvePictureImageURL = (value: string, sourceURL?: string) => {
+ if (!value?.trim()) {
+ return "";
+ }
+ try {
+ const url = sourceURL ? new URL(value, sourceURL) : new URL(value);
+ return /^https?:$/.test(url.protocol) ? url.href : "";
+ } catch (_error) {
+ return "";
+ }
+};
+
+// 保留 URL 内部的逗号,按候选描述符选择同一来源中的最大图片。
+export const getPictureImageSource = (srcset: string, sourceURL?: string) => {
+ let remaining = srcset;
+ let selected = "";
+ let selectedSize = 0;
+ while (remaining) {
+ remaining = remaining.replace(/^[\s,]+/, "");
+ const token = remaining.match(/^\S+/)?.[0];
+ if (!token) {
+ break;
+ }
+ remaining = remaining.substring(token.length);
+ let source = token;
+ let descriptor = "";
+ if (source.endsWith(",")) {
+ source = source.replace(/,+$/, "");
+ } else {
+ const end = remaining.indexOf(",");
+ descriptor = (end < 0 ? remaining : remaining.substring(0, end)).trim();
+ remaining = end < 0 ? "" : remaining.substring(end + 1);
+ }
+ if (descriptor && !/^(?:[1-9]\d*w|(?:\d+(?:\.\d+)?|\.\d+)x)$/.test(descriptor)) {
+ continue;
+ }
+ const size = descriptor ? Number.parseFloat(descriptor) : 1;
+ const url = resolvePictureImageURL(source, sourceURL);
+ if (url && Number.isFinite(size) && size > selectedSize) {
+ selected = url;
+ selectedSize = size;
+ }
+ }
+ return selected;
+};
+
+// 在收集附件前展开 picture 的真实来源,避免将其回退图片当成内嵌附件上传。
+export const normalizePictureImages = (root: ParentNode, sourceURL?: string,
+ sourceImages?: HTMLImageElement[]) => {
+ root.querySelectorAll<HTMLImageElement>("img").forEach((image, index) => {
+ const original = sourceImages?.[index] || image;
+ const picture = original.closest("picture");
+ if (!picture) {
+ return;
+ }
+ let source = resolvePictureImageURL(original.currentSrc, sourceURL);
+ const src = image.getAttribute("src")?.trim() || "";
+ if (!source && src && !/^data:image\//i.test(src)) {
+ return;
+ }
+ if (!source) {
+ for (const element of Array.from(picture.children)) {
+ if (element.tagName !== "SOURCE") {
+ continue;
+ }
+ const type = element.getAttribute("type")?.trim().toLowerCase();
+ if (type && !/^image\/(?:avif|webp|png|jpeg|gif|svg\+xml)$/.test(type)) {
+ continue;
+ }
+ const media = element.getAttribute("media");
+ if (media && !window.matchMedia(media).matches) {
+ continue;
+ }
+ source = getPictureImageSource(element.getAttribute("srcset") ||
+ element.getAttribute("data-srcset") || "", sourceURL);
+ if (source) {
+ break;
+ }
+ }
+ }
+ source = source || getPictureImageSource(original.getAttribute("srcset") ||
+ original.getAttribute("data-srcset") || "", sourceURL);
+ if (source) {
+ image.setAttribute("src", source);
+ // 清除会覆盖已选来源的懒加载属性,后续转换与资源下载使用同一地址。
+ ["srcset", "data-src", "data-srcset", "data-original", "data-origin-src", "data-lazy-src"].forEach(attribute =>
+ image.removeAttribute(attribute));
+ }
+ });
+};
diff --git a/app/tests/pictureImages.test.js b/app/tests/pictureImages.test.js
new file mode 100644
index 0000000000..59b1d5c7c7
--- /dev/null
+++ b/app/tests/pictureImages.test.js
@@ -0,0 +1,129 @@
+const assert = require("node:assert/strict");
+const {readFileSync, mkdtempSync, rmSync} = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+const runCases = async (sources) => {
+ const assert = require("node:assert/strict");
+ const modules = new Map();
+ const load = (name) => {
+ if (!modules.has(name)) {
+ const module = {exports: {}};
+ modules.set(name, module);
+ new Function("require", "exports", "module", sources[name])(load, module.exports, module);
+ }
+ return modules.get(name).exports;
+ };
+ const {normalizePictureImages} = load("./pictureImages");
+ const {hasHTMLEmbeddedAssets, collectHTMLEmbeddedAssets} = load("./htmlEmbeddedAssets");
+ const {collectHTMLLocalAssets} = load("./htmlLocalAssets");
+ const placeholder = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScLbtAAAAABJRU5ErkJggg==";
+ const root = document.createElement("template");
+ const sourceURL = "https://glyphsapp.com/zh/learn/vertical-metrics";
+ const realURL = "https://glyphsapp.com/media/verticaladobe-1280.webp";
+ // 评论截图中的来源分布:noscript 在 picture 前,实际图片仅存在于 source。
+ root.innerHTML = `<figure class="Image Image--shadow"><noscript><img src="${realURL}"></noscript>` +
+ `<picture class="js-only"><source type="image/webp" media="all" data-srcset="${realURL} 640w" ` +
+ `srcset="https://glyphsapp.com/media/verticaladobe-320.webp 160w, ${realURL} 640w">` +
+ `<img alt="diagram" data-lazyload data-sizes="90vw" src="${placeholder}" width="489" height="335"></picture></figure>`;
+ assert.equal(hasHTMLEmbeddedAssets(root.content), true);
+ normalizePictureImages(root.content, sourceURL);
+ assert.equal(root.content.querySelector("picture img").getAttribute("src"), realURL);
+ assert.equal(hasHTMLEmbeddedAssets(root.content), false);
+ assert.equal(collectHTMLEmbeddedAssets(root.content).length, 0);
+ assert.equal(collectHTMLLocalAssets(root.content).length, 0);
+ const blockDOM = globalThis.Lute.New().HTML2BlockDOM(root.innerHTML);
+ assert.ok(blockDOM.includes(realURL));
+ assert.equal(blockDOM.includes(placeholder), false);
+
+ root.innerHTML = "<picture><source srcset=\"/mobile.png 640w\" media=\"(max-width: 1px)\">" +
+ "<source type=\"image/jxl\" srcset=\"/unsupported.jxl 640w\">" +
+ `<source type="image/png" data-srcset="../images/a.png 640w"><img src="${placeholder}"></picture>`;
+ normalizePictureImages(root.content, sourceURL);
+ assert.equal(root.content.querySelector("img").getAttribute("src"), "https://glyphsapp.com/zh/images/a.png");
+
+ root.innerHTML = "<picture><source media=\"(max-width: 600px)\" srcset=\"https://example.com/mobile.png 320w\">" +
+ `<source srcset="https://example.com/desktop.png 1280w"><img src="${placeholder}"></picture>`;
+ normalizePictureImages(root.content, sourceURL);
+ assert.equal(root.content.querySelector("img").getAttribute("src"),
+ window.matchMedia("(max-width: 600px)").matches ? "https://example.com/mobile.png" : "https://example.com/desktop.png");
+
+ for (const html of [`<img src="${placeholder}">`, `<picture><img src="${placeholder}"></picture>`,
+ "<picture><source srcset=\"https://example.com/real.png 640w\"><img src=\"assets/existing.png\"></picture>"]) {
+ root.innerHTML = html;
+ const before = root.innerHTML;
+ normalizePictureImages(root.content, sourceURL);
+ assert.equal(root.innerHTML, before);
+ }
+
+ // 选区只包含 img 时仍使用原页面选中的来源,并保持原页面属性不变。
+ root.innerHTML = "<picture><source srcset=\"https://example.com/large.png 640w\">" +
+ `<img src="${placeholder}" data-original="https://example.com/placeholder.png"></picture>`;
+ const original = root.content.querySelector("img");
+ Object.defineProperty(original, "currentSrc", {value: "https://example.com/selected.png"});
+ const originalHTML = root.innerHTML;
+ const range = document.createRange();
+ range.selectNode(original);
+ const selected = document.createElement("div");
+ selected.append(range.cloneContents());
+ normalizePictureImages(selected, sourceURL, [original]);
+ assert.equal(selected.querySelector("img").getAttribute("src"), original.currentSrc);
+ assert.equal(selected.querySelector("img").hasAttribute("data-original"), false);
+ assert.equal(root.innerHTML, originalHTML);
+ normalizePictureImages(selected, sourceURL);
+ assert.equal(selected.querySelector("img").getAttribute("src"), original.currentSrc);
+};
+
+const runElectron = async () => {
+ const {app, BrowserWindow} = require("electron");
+ app.setPath("userData", process.argv[2]);
+ app.commandLine.appendSwitch("disable-gpu");
+ await app.whenReady();
+ const win = new BrowserWindow({show: false, webPreferences: {nodeIntegration: true, contextIsolation: false}});
+ let exitCode = 0;
+ try {
+ const ts = require("typescript");
+ const sources = Object.fromEntries([
+ ["pictureImages", "util"], ["htmlEmbeddedAssets", "upload"], ["htmlLocalAssets", "upload"], ["base64File", "upload"],
+ ].map(([name, directory]) => ["./" + name,
+ ts.transpileModule(readFileSync(path.join(__dirname, `../src/protyle/${directory}/${name}.ts`), "utf8"),
+ {compilerOptions: {module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020}}).outputText]));
+ await win.loadURL("data:text/html,<html><body></body></html>");
+ await win.webContents.executeJavaScript(readFileSync(path.join(__dirname, "../stage/protyle/js/lute/lute.min.js"), "utf8"));
+ await win.webContents.executeJavaScript(`(${runCases.toString()})(${JSON.stringify(sources)})`);
+ win.setSize(480, 800);
+ await win.webContents.executeJavaScript(`(${runCases.toString()})(${JSON.stringify(sources)})`);
+ console.log("Picture image paste cases passed");
+ } catch (error) {
+ console.error(error);
+ exitCode = 1;
+ } finally {
+ win.destroy();
+ app.exit(exitCode);
+ }
+};
+
+if (process.versions.electron && process.type === "browser") {
+ runElectron().catch(error => {
+ console.error(error);
+ require("electron").app.exit(1);
+ });
+} else {
+ require("node:test").it("normalizes picture images before embedded uploads and Lute conversion using browser DOM", {
+ skip: process.platform === "linux" && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY,
+ timeout: 45000,
+ }, async () => {
+ const env = {...process.env};
+ delete env.ELECTRON_RUN_AS_NODE;
+ const profile = mkdtempSync(path.join(os.tmpdir(), "siyuan-picture-images-"));
+ try {
+ const {stdout} = await require("node:util").promisify(require("node:child_process").execFile)(
+ require("electron"), [__filename, profile], {env, windowsHide: true, timeout: 40000});
+ assert.match(stdout, /Picture image paste cases passed/);
+ } finally {
+ assert.equal(path.dirname(path.resolve(profile)), path.resolve(os.tmpdir()));
+ assert.ok(path.basename(profile).startsWith("siyuan-picture-images-"));
+ rmSync(profile, {recursive: true, force: true, maxRetries: 5, retryDelay: 100});
+ }
+ });
+}