YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(notes, html): keep pasted images intact and stop the link filter blanking pages
Manas Raghuwanshic81ad7e3475f
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/image-utils.test.ts b/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/image-utils.test.tsnew file mode 100644index 000000000..fade3610f--- /dev/null+++ b/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/image-utils.test.ts@@ -0,0 +1,97 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { afterEach, describe, expect, it, vi } from "vitest";+import { resizeImageDataUrl } from "../image-utils";++// jsdom decodes no images, so the browser's Image and canvas are stood in for:+// an image "decodes" to the size given for its data URL, or fails without one.+const sizes = new Map<string, { width: number; height: number }>();+const drawn: string[] = [];++class FakeImage {+ width = 0;+ height = 0;+ onload: (() => void) | null = null;+ onerror: (() => void) | null = null;+ set src(value: string) {+ const size = sizes.get(value);+ queueMicrotask(() => {+ if (!size) return this.onerror?.();+ Object.assign(this, size);+ this.onload?.();+ });+ }+}++function fakeCanvas() {+ const canvas = { width: 0, height: 0 } as HTMLCanvasElement;+ const ctx = {+ set fillStyle(color: string) {+ drawn.push(`fill ${color}`);+ },+ fillRect: () => drawn.push("fillRect"),+ drawImage: () => drawn.push(`draw ${canvas.width}x${canvas.height}`),+ };+ canvas.getContext = (() => ctx) as unknown as HTMLCanvasElement["getContext"];+ canvas.toDataURL = (type?: string) => `data:${type};base64,REENCODED`;+ return canvas;+}++function image(type: string, length: number, size?: { width: number; height: number }) {+ const prefix = `data:${type};base64,`;+ const dataUrl = prefix + "A".repeat(length - prefix.length);+ if (size) sizes.set(dataUrl, size);+ return dataUrl;+}++describe("resizeImageDataUrl", () => {+ vi.stubGlobal("Image", FakeImage);+ const createElement = document.createElement.bind(document);+ vi.spyOn(document, "createElement").mockImplementation((tag: string) =>+ tag === "canvas" ? fakeCanvas() : createElement(tag),+ );++ afterEach(() => {+ sizes.clear();+ drawn.length = 0;+ });++ it("keeps a small image that fits as it came, with its transparency and animation", async () => {+ for (const type of ["image/png", "image/gif", "image/webp"]) {+ const original = image(type, 40_000, { width: 300, height: 200 });+ await expect(resizeImageDataUrl(original)).resolves.toBe(original);+ }+ expect(drawn).toEqual([]);+ });++ it("re-encodes a large image onto white, never onto black", async () => {+ const result = await resizeImageDataUrl(image("image/png", 900_000, { width: 800, height: 600 }));+ expect(result).toBe("data:image/jpeg;base64,REENCODED");+ expect(drawn).toEqual(["fill #fff", "fillRect", "draw 800x600"]);+ });++ it("scales an image wider than the note allows", async () => {+ await resizeImageDataUrl(image("image/png", 40_000, { width: 4096, height: 1024 }));+ expect(drawn.at(-1)).toBe("draw 1024x256");+ });++ it("keeps a small SVG at any size, and re-encodes a large one", async () => {+ const small = image("image/svg+xml", 40_000, { width: 5000, height: 5000 });+ await expect(resizeImageDataUrl(small)).resolves.toBe(small);+ const large = image("image/svg+xml", 900_000, { width: 500, height: 500 });+ await expect(resizeImageDataUrl(large)).resolves.toBe("data:image/jpeg;base64,REENCODED");+ });++ it("returns null for an image that can't be decoded, of any size", async () => {+ await expect(resizeImageDataUrl(image("image/png", 40_000))).resolves.toBeNull();+ await expect(resizeImageDataUrl(image("image/svg+xml", 40_000))).resolves.toBeNull();+ await expect(resizeImageDataUrl(image("image/png", 900_000))).resolves.toBeNull();+ });++ it("returns null for a large image with no size to scale from", async () => {+ await expect(+ resizeImageDataUrl(image("image/svg+xml", 900_000, { width: 0, height: 0 })),+ ).resolves.toBeNull();+ });+});diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-remote-images.test.ts b/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-remote-images.test.tsindex 7e287cc30..a31588367 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-remote-images.test.ts+++ b/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-remote-images.test.ts@@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";import { Editor } from "@tiptap/core";const tauriFetchMock = vi.hoisted(() => vi.fn());+const resizeMock = vi.hoisted(() => vi.fn(async (dataUrl: string): Promise<string | null> => dataUrl));vi.mock("@/lib/http/tauri-fetch", () => ({tauriFetchWithDeadline: tauriFetchMock,@@ -13,7 +14,7 @@ vi.mock("@/lib/http/tauri-fetch", () => ({// jsdom cannot decode images, so resizing would never settle.vi.mock("../image-utils", async (importOriginal) => ({...(await importOriginal<typeof import("../image-utils")>()),- resizeImageDataUrl: async (dataUrl: string) => dataUrl,+ resizeImageDataUrl: resizeMock,}));import {@@ -141,6 +142,7 @@ describe("meeting note remote images", () => {const before = getMarkdown(editor);tauriFetchMock.mockResolvedValue(imageResponse());expect(tauriFetchMock).not.toHaveBeenCalled();+ expect(loadButton(mounted.element)).toHaveTextContent("Load image from example.com");loadButton(mounted.element).click();@@ -176,6 +178,28 @@ describe("meeting note remote images", () => {await vi.waitFor(() => expect(getMarkdown(editor!)).toBe(""));});+ it("keeps the chip when the download is not an image the webview can show", async () => {+ const mounted = mountEditor("");+ editor = mounted.editor;+ tauriFetchMock.mockResolvedValue(imageResponse());+ resizeMock.mockResolvedValueOnce(null);++ loadButton(mounted.element).click();++ await vi.waitFor(() => expect(loadButton(mounted.element)).toHaveTextContent("Couldn't load image. Try again"));+ expect(mounted.element.querySelector("img")).toBeNull();+ expect(getMarkdown(editor)).toBe("");+ });++ it("names the host the download goes to, not what the alt text claims", () => {+ const mounted = mountEditor(+ '',+ );+ editor = mounted.editor;++ expect(loadButton(mounted.element)).toHaveTextContent(/^Load image from evil\.example:8443$/);+ });+it("does not change the note when the image changed during the download", async () => {let respond!: (response: Response) => void;tauriFetchMock.mockReturnValue(new Promise<Response>((resolve) => (respond = resolve)));diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-roundtrip.test.ts b/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-roundtrip.test.tsindex 13b66e363..22678fc16 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-roundtrip.test.ts+++ b/apps/screenpipe-app-tauri/components/meeting-notes/__tests__/note-editor-roundtrip.test.ts@@ -105,4 +105,15 @@ describe("meeting note markdown round-trip (#5369)", () => {expect(roundtrip(once)).toBe(once);}});++ it("a block after an image keeps its own line", () => {+ for (const md of [+ "Notes\n\n\n\n## Summary",+ "\n\n- follow up",+ '<img src="data:image/png;base64,AAAA" width="20" />\n\n## Summary',+ "\n\nplain text",+ ]) {+ expect(roundtrip(md)).toBe(md);+ }+ });});diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/image-utils.ts b/apps/screenpipe-app-tauri/components/meeting-notes/image-utils.tsindex 8c3bb2dd8..ed7acd0f7 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/image-utils.ts+++ b/apps/screenpipe-app-tauri/components/meeting-notes/image-utils.ts@@ -24,6 +24,9 @@ const IMAGE_MIME_BY_EXT: Record<string, string> = {const MAX_IMAGE_EDGE_PX = 1024;const JPEG_QUALITY = 0.82;+// The largest image a note keeps as it came. Notes save images inline and+// serialize them on every edit, so anything larger is re-encoded.+const MAX_KEPT_IMAGE_DATA_URL_LENGTH = 512 * 1024;export function imageExtensionFromName(name: string): string {return name.split(".").pop()?.toLowerCase() ?? "";@@ -60,18 +63,30 @@ export function imageBytesToDataUrl(return `data:${mime};base64,${btoa(binary)}`;}-export function resizeImageDataUrl(dataUrl: string): Promise<string> {+/**+ * Readies an image for a note, which saves it inline. A small image that fits+ * stays as it came, keeping its transparency and animation (an SVG scales by+ * itself, so only its size counts); any other is scaled to fit and re-encoded+ * as JPEG. Returns null for an image the webview can't decode, so no broken+ * picture is saved into a note.+ */+export function resizeImageDataUrl(dataUrl: string): Promise<string | null> {if (!dataUrl.startsWith("data:image/")) return Promise.resolve(dataUrl);- if (dataUrl.startsWith("data:image/svg+xml")) return Promise.resolve(dataUrl);+ const small = dataUrl.length <= MAX_KEPT_IMAGE_DATA_URL_LENGTH;+ const isSvg = dataUrl.startsWith("data:image/svg+xml");return new Promise((resolve) => {const img = new Image();img.onload = () => {let { width, height } = img;- if (!width || !height) {+ if (small && (isSvg || (width <= MAX_IMAGE_EDGE_PX && height <= MAX_IMAGE_EDGE_PX))) {resolve(dataUrl);return;}+ if (!width || !height) {+ resolve(null);+ return;+ }if (width > MAX_IMAGE_EDGE_PX || height > MAX_IMAGE_EDGE_PX) {const scale = MAX_IMAGE_EDGE_PX / Math.max(width, height);@@ -85,16 +100,19 @@ export function resizeImageDataUrl(dataUrl: string): Promise<string> {canvas.height = height;const ctx = canvas.getContext("2d");if (!ctx) {- resolve(dataUrl);+ resolve(null);return;}+ // JPEG has no transparency, and a transparent pixel would turn black.+ ctx.fillStyle = "#fff";+ ctx.fillRect(0, 0, width, height);ctx.drawImage(img, 0, 0, width, height);resolve(canvas.toDataURL("image/jpeg", JPEG_QUALITY));} catch {- resolve(dataUrl);+ resolve(null);}};- img.onerror = () => resolve(dataUrl);+ img.onerror = () => resolve(null);img.src = dataUrl;});}diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/note-editor.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/note-editor.tsxindex 7fcb5f4b0..c6c9ae99b 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/note-editor.tsx+++ b/apps/screenpipe-app-tauri/components/meeting-notes/note-editor.tsx@@ -44,15 +44,19 @@ function isWebImageSource(src: unknown): src is string {}export interface MeetingNoteImageLabels {- /** The button that downloads a blocked web image into the note. */- load: string;+ /**+ * The button that downloads a blocked web image into the note. It names the+ * host the download goes to: the alt text beside it was written by whoever+ * wrote the note, and could ask for the click.+ */+ load: (host: string) => string;loading: string;/** The same button after a download failed. */retry: string;}const ENGLISH_IMAGE_LABELS: MeetingNoteImageLabels = {- load: "Load image",+ load: (host) => `Load image from ${host}`,loading: "Loading image…",retry: "Couldn't load image. Try again",};@@ -81,6 +85,14 @@ function blockedImageSpec(attrs: Record<string, unknown>): DOMOutputSpec {];}+function imageHost(src: string): string {+ try {+ return new URL(src).host;+ } catch {+ return src;+ }+}+// How the note shows an image it doesn't carry. A web image gets a button// that downloads it into the note once the user asks, the way a paste does;// the note then shows it like any embedded image. Nothing loads before that.@@ -88,13 +100,14 @@ function blockedImageView(props: NodeViewRendererProps, labels: MeetingNoteImageconst dom = DOMSerializer.renderSpec(document, blockedImageSpec(props.node.attrs)).dom as HTMLElement;const src = props.node.attrs.src;if (!isWebImageSource(src)) return { dom };+ const loadLabel = labels.load(imageHost(src));const button = document.createElement("button");button.type = "button";button.dataset.imageLoad = "";button.className =- "shrink-0 cursor-pointer rounded px-1.5 py-0.5 font-medium text-foreground underline-offset-2 hover:underline disabled:cursor-default disabled:no-underline disabled:text-muted-foreground";- button.textContent = labels.load;+ "min-w-0 cursor-pointer text-left [overflow-wrap:anywhere] rounded px-1.5 py-0.5 font-medium text-foreground underline-offset-2 hover:underline disabled:cursor-default disabled:no-underline disabled:text-muted-foreground";+ button.textContent = loadLabel;// Keep the caret where it was.button.addEventListener("mousedown", (event) => event.preventDefault());button.addEventListener("click", async () => {@@ -106,7 +119,7 @@ function blockedImageView(props: NodeViewRendererProps, labels: MeetingNoteImage// The note turned read-only meanwhile (a summary is being written into// it), so it takes no edit; the chip offers the download again.if (!props.editor.isEditable) {- button.textContent = labels.load;+ button.textContent = loadLabel;return;}const pos = props.getPos();@@ -170,6 +183,9 @@ const ResizableImage = Image.extend<ImageOptions & { labels: MeetingNoteImageLab`.replace(/[()]/g, "\\$&")}${title ? ` "${title.replace(/"/g, '\\"')}"` : ""})`,);}+ // A block image ends its block, or the next block (a heading,+ // a list) is glued onto the image's line and turns into text.+ state.closeBlock(node);},parse: {// markdown-it handles both  and <img> natively@@ -562,7 +578,7 @@ function NoteEditor(immediatelyRender: false,editable: !readOnly,extensions: createMeetingNoteEditorExtensions(placeholder ?? "", {- load: ui("Load image"),+ load: (host) => ui("Load image from {host}", { host }),loading: ui("Loading image…"),retry: ui("Couldn't load image. Try again"),}),diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsx b/apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsxindex e36316fd6..4e59e43d7 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsx+++ b/apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsx@@ -836,7 +836,8 @@ export function NoteView({const images: string[] = [];for (const path of imagePaths) {const raw = imageBytesToDataUrl(path, await readFile(path));- if (raw) images.push(await resizeImageDataUrl(raw));+ const image = raw && (await resizeImageDataUrl(raw));+ if (image) images.push(image);}if (images.length === 0) return;const { x, y } = toClient(pos);diff --git a/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-diagram.test.tsx b/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-diagram.test.tsxindex df91f36ea..f84298e76 100644--- a/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-diagram.test.tsx+++ b/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-diagram.test.tsx@@ -42,7 +42,9 @@ describe("MermaidDiagram", () => {// The remote URLs exist only inside the inert image data.expect(container.querySelectorAll("svg, foreignObject, image")).toHaveLength(0);expect(container.querySelectorAll("img")).toHaveLength(1);- expect(container.innerHTML).not.toContain("https://evil.example");+ expect(decodeURIComponent(image.getAttribute("src")!)).toContain("https://evil.example/label.png");+ image.removeAttribute("src");+ expect(container.innerHTML).not.toContain("evil.example");});it("shows the source when the diagram cannot be drawn", async () => {diff --git a/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-sandbox.test.ts b/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-sandbox.test.tsindex 702234750..41f4989e0 100644--- a/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-sandbox.test.ts+++ b/apps/screenpipe-app-tauri/components/rewind/__tests__/mermaid-sandbox.test.ts@@ -16,6 +16,15 @@ window.mermaid = {window.mermaidCalls.documents.push(document);if (chart === "bad") return Promise.reject(new Error("Unsupported color format"));if (chart === "never") return new Promise(function () {});+ if (chart === "image shape") {+ // Mermaid sizes an image shape from its decoded picture.+ var picture = new Image();+ picture.src = "https://evil.example/shape.png";+ return picture.decode().then(function () {+ window.mermaidCalls.decoded = picture.src;+ return { svg: '<svg viewBox="0 0 10 10"><text>image shape</text></svg>' };+ });+ }return Promise.resolve({svg: '<svg id="' + id + '" width="100%" style="max-width: 200px;" viewBox="0 0 200 100">' +'<style>.node { fill: red; }</style>' +@@ -25,7 +34,7 @@ window.mermaid = {};`;type FrameWindow = Window & {- mermaidCalls: { configs: Array<Record<string, unknown>>; documents: Document[] };+ mermaidCalls: { configs: Array<Record<string, unknown>>; documents: Document[]; decoded?: string };};describe("renderMermaidSvg", () => {@@ -124,6 +133,32 @@ describe("renderMermaidSvg", () => {}});+ it("draws an image shape whose picture the CSP blocks from a blank stand-in", async () => {+ // jsdom decodes nothing. Give each new frame the browser's behavior: the+ // CSP refuses a web picture, so its decode rejects; a data: picture decodes.+ const observer = new MutationObserver(() => {+ const frame = document.querySelector("iframe");+ const FrameImage = (frame?.contentWindow as unknown as typeof globalThis | null)?.HTMLImageElement;+ if (!FrameImage || FrameImage.prototype.decode) return;+ FrameImage.prototype.decode = function (this: HTMLImageElement) {+ return this.src.startsWith("data:")+ ? Promise.resolve()+ : Promise.reject(new Error("The source image cannot be decoded."));+ };+ });+ observer.observe(document.body, { childList: true });+ try {+ const { renderMermaidSvg } = await import("../mermaid-sandbox");++ await expect(renderMermaidSvg("image shape", {})).resolves.toMatchObject({ text: "image shape" });++ const frame = document.querySelector("iframe")!;+ expect((frame.contentWindow as FrameWindow).mermaidCalls.decoded).toMatch(/^data:image\/png;base64,/);+ } finally {+ observer.disconnect();+ }+ });+it("reports a bundle that fails to load and tries again next time", async () => {const { renderMermaidSvg } = await import("../mermaid-sandbox");fetchMock.mockResolvedValueOnce(new Response("missing", { status: 404 }));diff --git a/apps/screenpipe-app-tauri/lib/__tests__/markdown-remote-images.test.ts b/apps/screenpipe-app-tauri/lib/__tests__/markdown-remote-images.test.tsindex b51fe7685..db2c57503 100644--- a/apps/screenpipe-app-tauri/lib/__tests__/markdown-remote-images.test.ts+++ b/apps/screenpipe-app-tauri/lib/__tests__/markdown-remote-images.test.ts@@ -71,6 +71,15 @@ describe("remote images in rendered content", () => {).toEqual(REVIEWED_MARKDOWN_RENDERERS);});+ it("parses raw HTML in markdown only where the chat sanitizer follows", () => {+ expect(+ filesImporting("rehype-raw"),+ "Raw HTML in markdown brings tags the img component never sees, such as " ++ "<video poster>. Render it through MarkdownBlock in components/chat/markdown-block.tsx, " ++ "which sanitizes what rehype-raw parses, or sanitize it the same way and add it here.",+ ).toEqual(["components/chat/markdown-block.tsx"]);+ });+it("never loads Mermaid into the app window", () => {expect(filesImporting("mermaid"),diff --git a/apps/screenpipe-app-tauri/lib/utils/html-sandbox.test.ts b/apps/screenpipe-app-tauri/lib/utils/html-sandbox.test.tsindex 7f666e8b3..e1836e893 100644--- a/apps/screenpipe-app-tauri/lib/utils/html-sandbox.test.ts+++ b/apps/screenpipe-app-tauri/lib/utils/html-sandbox.test.ts@@ -156,9 +156,23 @@ describe("withoutLinkElements", () => {['<lin<link>k rel="preconnect" href="https://a.example">x', "x"],['<link title=">" rel="preconnect" href="https://a.example">x', '" rel="preconnect" href="https://a.example">x'],["<linkish>kept</linkish>", "<linkish>kept</linkish>"],+ ["<link/rel=preconnect href=//a.example>x", "x"],+ ["<link\trel=preconnect\fhref=//a.example>x", "x"],+ ["<link>x", "x"],])("removes every link from %j", (input, expected) => {expect(withoutLinkElements(input)).toBe(expected);});++ it.each([+ // A custom element whose name starts with link-.+ '<link-card href="/a">Docs</link-card><p>after</p>',+ // Script that only mentions links. Removing up to the next ">" would eat+ // the </script> and blank the page.+ "<script>for(var i=0;i<link.length;i++){}</script><h1>Report</h1>",+ "<script>const parts = html.split('<link');</script><h1>Report</h1>",+ ])("leaves %j alone", (input) => {+ expect(withoutLinkElements(input)).toBe(input);+ });});describe("wrapHtmlForSandbox", () => {diff --git a/apps/screenpipe-app-tauri/lib/utils/html-sandbox.ts b/apps/screenpipe-app-tauri/lib/utils/html-sandbox.tsindex 9038a187c..31af1a2c1 100644--- a/apps/screenpipe-app-tauri/lib/utils/html-sandbox.ts+++ b/apps/screenpipe-app-tauri/lib/utils/html-sandbox.ts@@ -147,16 +147,20 @@ parent.postMessage({source:TAG,type:'openLink',url:href},'*');})();</script>`;/**- * Removes every `<link>` from artifact HTML. The CSP blocks what a link loads,- * except WebKit's connection hints (`preconnect`, `dns-prefetch`), which reach- * their host anyway, and the host name alone can carry data out. Repeats until- * nothing changes, so a removal can't join the text around it into a new link.+ * Removes every `<link>` tag from artifact HTML. The CSP blocks what a link+ * loads, except WebKit's connection hints (`preconnect`, `dns-prefetch`),+ * which reach their host anyway, and the host name alone can carry data out.+ * Only a tag name the parser would read as `link` matches (`<link` then a+ * space, `/` or `>`), so script such as `i<link.length` and elements such as+ * `<link-card>` stay intact. Repeats until nothing changes, so a removal can't+ * join the text around it into a new link. This covers the markup only: where+ * scripts run (the full viewer), a script can still add a link itself.*/export function withoutLinkElements(html: string): string {let previous: string;do {previous = html;- html = html.replace(/<link\b[^>]*>?/gi, "");+ html = html.replace(/<link(?=[\s/>])[^>]*>?/gi, "");} while (html !== previous);return html;}