YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(markdown): stop rendered markdown from loading remote images
Manas Raghuwanshi7ee2e6ef0782
diff --git a/apps/screenpipe-app-tauri/app/notification-panel/page.test.tsx b/apps/screenpipe-app-tauri/app/notification-panel/page.test.tsxindex d3853642d..57365ad99 100644--- a/apps/screenpipe-app-tauri/app/notification-panel/page.test.tsx+++ b/apps/screenpipe-app-tauri/app/notification-panel/page.test.tsx@@ -119,6 +119,19 @@ describe("compact notification panel", () => {expect(bridge.hide).toHaveBeenCalledTimes(1);});+ it("shows a remote image in the body as alt text without loading it", async () => {+ const { container } = render(<NotificationPanelPage />);+ await deliver({+ ...payload,+ body: "chart below ",+ });+ // The panel's own app icon is an <img>, so check only the body.+ const body = container.querySelector(".notif-md");+ expect(body).toHaveTextContent("chart below remote chart");+ expect(body?.querySelector("img")).toBeNull();+ expect(container.querySelector("img[src*='example.com']")).toBeNull();+ });+it("pauses expiration while options are open and resumes when closed", async () => {vi.useFakeTimers();render(<NotificationPanelPage />);diff --git a/apps/screenpipe-app-tauri/app/notification-panel/page.tsx b/apps/screenpipe-app-tauri/app/notification-panel/page.tsxindex 556315f2f..e25bf88c7 100644--- a/apps/screenpipe-app-tauri/app/notification-panel/page.tsx+++ b/apps/screenpipe-app-tauri/app/notification-panel/page.tsx@@ -660,6 +660,9 @@ export default function NotificationPanelPage() {</>);},+ // Bodies come from pipes; a remote image would load on render+ // and send its URL out, so show the alt text instead.+ img: ({ alt }) => (alt ? <span>{alt}</span> : null),}}>{payload.body}diff --git a/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx b/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsxindex aa447709d..db9f726d5 100644--- a/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx+++ b/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx@@ -90,18 +90,74 @@ describe("MemoizedReactMarkdown local images", () => {expect(container.querySelector("img")).toBeNull();});- it("leaves remote https images on their original src", () => {+ // A remote image loads the moment it renders and sends its URL to that+ // server, so markdown shows the alt text instead of fetching anything.+ it.each([+ ["https", ""],+ ["http", ""],+ ["protocol-relative", ""],+ ["reference-style", "![remote chart][r]\n\n[r]: https://example.com/pipe.png"],+ ["relative", ""],+ ])("shows %s images as alt text without loading them", (_name, markdown) => {+ const { container } = render(+ <MemoizedReactMarkdown urlTransform={chatUrlTransform}>+ {markdown}+ </MemoizedReactMarkdown>,+ );++ expect(screen.getByText("remote chart")).toBeInTheDocument();+ expect(container.querySelector("img")).toBeNull();+ expect(getMediaFileMock).not.toHaveBeenCalled();+ });++ it("does not load remote images without a url transform or alt text", () => {+ // Daily summaries and meeting notes use react-markdown's default transform.+ const { container } = render(+ <MemoizedReactMarkdown>+ {"before  after "}+ </MemoizedReactMarkdown>,+ );++ expect(container).toHaveTextContent("before after remote chart");+ expect(container.querySelector("img")).toBeNull();+ });++ it.each([+ ["remote media image", "", () => screen.getByText("clip")],+ [+ "remote media link",+ "[clip](https://example.com/a.mp4)",+ () => expect(screen.getByRole("link", { name: "clip" })).toHaveAttribute("href", "https://example.com/a.mp4"),+ ],+ ["remote media code", "`https://example.com/a.mp4`", () => screen.getByText("https://example.com/a.mp4")],+ ["network-share media image", "", () => screen.getByText("clip")],+ ["network-share media code", "`\\\\host\\share\\a.mp4`", () => screen.getByText("\\\\host\\share\\a.mp4")],+ ])("keeps %s away from the native media reader", (_name, markdown, expectOrdinaryForm) => {render(<MemoizedReactMarkdown urlTransform={chatUrlTransform}>- {""}+ {markdown}</MemoizedReactMarkdown>,);- expect(screen.getByAltText("remote")).toHaveAttribute(- "src",- "https://example.com/pipe.png",+ expectOrdinaryForm();+ expect(screen.queryByTestId("media-component")).toBeNull();+ });++ it.each([+ ["absolute", "", "/Users/me/a.mp4"],+ ["home-relative code", "`~/Downloads/clip.mp4`", "~/Downloads/clip.mp4"],+ ["Windows", "", "C:\\Users\\me\\a.mp4"],+ ["file URL link", "[clip](file:///Users/me/a.mp4)", "file:///Users/me/a.mp4"],+ ])("still plays %s local media", (_name, markdown, path) => {+ render(+ <MemoizedReactMarkdown urlTransform={chatUrlTransform}>+ {markdown}+ </MemoizedReactMarkdown>,);- expect(getMediaFileMock).not.toHaveBeenCalled();++ // Markdown percent-encodes backslashes; MediaComponent decodes them.+ const filePath = screen.getByTestId("media-component").textContent ?? "";+ expect(decodeURIComponent(filePath)).toBe(path);});it("recognizes Windows absolute image paths as local files", () => {@@ -112,6 +168,22 @@ describe("MemoizedReactMarkdown local images", () => {).toBe("C:\\Users\\Hugo\\.screenpipe\\data\\missing.jpg");});+ it.each([+ ["/tmp/x.png", "/tmp/x.png"],+ ["file:///tmp/x.png", "/tmp/x.png"],+ ["file:///C:/Users/me/x.png", "C:/Users/me/x.png"],+ ])("accepts local path %s", (url, path) => {+ expect(resolveLocalPathFromMarkdownUrl(url)).toBe(path);+ });++ // Windows reads these as network shares, which contacts the named host.+ it.each(["//host/share/x.png", "/\\host\\share\\x.png", "file:///%2F%2Fhost/x.png"])(+ "rejects network-share path %s",+ (url) => {+ expect(resolveLocalPathFromMarkdownUrl(url)).toBeNull();+ },+ );+it("keeps file:// image URLs so the local loader still receives a path", () => {expect(chatUrlTransform("file:///tmp/waterfall-light.gif")).toBe("file:///tmp/waterfall-light.gif",diff --git a/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx b/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsxindex aa635bb2d..f47eefe21 100644--- a/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx+++ b/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx@@ -17,7 +17,9 @@ const {setPendingNavigationMock,showWindowMock,routeNotificationDeeplinkMock,+ getMediaFileMock,} = vi.hoisted(() => ({+ getMediaFileMock: vi.fn(),ownedBrowserNavigateMock: vi.fn(async () => ({ status: "ok" as const, data: null })),openUrlMock: vi.fn(async () => undefined),copyTextToClipboardMock: vi.fn(async () => ({ status: "ok" as const, data: null })),@@ -41,6 +43,7 @@ vi.mock("@/lib/utils/tauri", () => ({}));vi.mock("@tauri-apps/plugin-opener", () => ({ openUrl: openUrlMock }));+vi.mock("@/lib/actions/video-actions", () => ({ getMediaFile: getMediaFileMock }));vi.mock("@/components/ui/use-toast", () => ({ toast: toastMock }));vi.mock("@tauri-apps/api/event", () => ({@@ -425,10 +428,49 @@ describe("MarkdownBlock", () => {it("neutralizes javascript: URLs in raw HTML attributes", () => {const container = renderHtml(- `<a href="javascript:alert(1)">click</a> <img src="https://example.com/x.png" longdesc="javascript:alert(1)">`,+ `<a href="javascript:alert(1)">click</a> <img src="https://example.com/x.png" alt="remote chart" longdesc="javascript:alert(1)">\n\n<blockquote cite="javascript:alert(1)">quoted</blockquote>`,);+ expect(screen.getByText("click")).toBeInTheDocument();expect(container.querySelector("a[href^='javascript']")).toBeNull();- expect(container.querySelector("img")?.getAttribute("longdesc")).toBeNull();+ expect(container.querySelector("img")).toBeNull();+ expect(container).toHaveTextContent("remote chart");+ const quote = container.querySelector("blockquote");+ expect(quote).toHaveTextContent("quoted");+ expect(quote?.getAttribute("cite") ?? "").not.toMatch(/javascript:/i);+ expect(container.innerHTML).not.toMatch(/javascript:/i);+ });++ // A remote image loads as soon as it renders and sends its URL (and any+ // data the AI was steered into putting there) to that server.+ it.each([+ ["markdown image", ``],+ ["raw img", `<img src="https://example.com/x.png?d=secret" alt="remote chart">`],+ ["protocol-relative raw img", `<img src="//example.com/x.png" alt="remote chart">`],+ ])("shows a remote %s as alt text without loading it", (_name, html) => {+ const container = renderHtml(`before\n\n${html}\n\nafter`);+ expect(container).toHaveTextContent("before");+ expect(container).toHaveTextContent("remote chart");+ expect(container).toHaveTextContent("after");+ expect(container.querySelector("img")).toBeNull();+ });++ it("drops picture/source so a srcset cannot replace a local image", async () => {+ getMediaFileMock.mockResolvedValue({+ data: "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",+ mimeType: "image/gif",+ });+ URL.createObjectURL = vi.fn(() => "blob:local-picture");+ URL.revokeObjectURL = vi.fn();++ const container = renderHtml(+ `before\n\n<picture><source srcset="https://example.com/x.png?d=secret"><img src="/tmp/local.gif" alt="pic"></picture>\n\nafter`,+ );++ expect(await screen.findByAltText("pic")).toHaveAttribute("src", "blob:local-picture");+ expect(getMediaFileMock).toHaveBeenCalledWith("/tmp/local.gif");+ expect(container.querySelector("picture, source, [srcset]")).toBeNull();+ expect(container).toHaveTextContent("before");+ expect(container).toHaveTextContent("after");});it("keeps safe formatting and collapsible details", () => {diff --git a/apps/screenpipe-app-tauri/components/chat/markdown-block.tsx b/apps/screenpipe-app-tauri/components/chat/markdown-block.tsxindex 00442537c..f5ce9325c 100644--- a/apps/screenpipe-app-tauri/components/chat/markdown-block.tsx+++ b/apps/screenpipe-app-tauri/components/chat/markdown-block.tsx@@ -38,10 +38,19 @@ import { useGT } from "gt-react";// which react-markdown applies after rehype and which already allows// screenpipe:// and local file links that the default protocol list would// strip; attributes urlTransform never sees keep the default protocol check.+// <picture>/<source> are dropped: srcset is one of those attributes, has no+// protocol rule, and would make the browser fetch a remote image in place of+// a local <img> next to it. Images render only through the shared img+// component, which loads local files and nothing else.const CLOBBER_PREFIX = "user-content-";const chatSanitizeSchema = {...defaultSchema,- tagNames: [...(defaultSchema.tagNames ?? []), "u", "mark", "small"],+ tagNames: [+ ...(defaultSchema.tagNames ?? []).filter((tag) => tag !== "picture" && tag !== "source"),+ "u",+ "mark",+ "small",+ ],protocols: { ...defaultSchema.protocols, href: [], src: [] },clobberPrefix: CLOBBER_PREFIX,strip: [...(defaultSchema.strip ?? []), "style"],diff --git a/apps/screenpipe-app-tauri/components/markdown.tsx b/apps/screenpipe-app-tauri/components/markdown.tsxindex 0e25070df..83bdd0556 100644--- a/apps/screenpipe-app-tauri/components/markdown.tsx+++ b/apps/screenpipe-app-tauri/components/markdown.tsx@@ -26,10 +26,8 @@ export function resolveLocalPathFromMarkdownUrl(url: string): string | null {const urlWithoutFragment = raw.split("#", 1)[0] ?? raw;let candidate = urlWithoutFragment;- let wasFileUri = false;if (/^file:\/\//i.test(candidate)) {- wasFileUri = true;const withoutScheme = candidate.replace(/^file:\/\//i, "");candidate = `/${withoutScheme.replace(/^\/+/, "")}`;}@@ -44,7 +42,9 @@ export function resolveLocalPathFromMarkdownUrl(url: string): string | null {candidate = candidate.slice(1);}- if (candidate.startsWith("/") && (wasFileUri || !candidate.startsWith("//"))) {+ // A second leading slash or backslash is a network share on Windows+ // (//host/share, /\host\share); reading it would contact that host.+ if (/^\/(?![\\/])/.test(candidate)) {return candidate;}@@ -55,6 +55,14 @@ export function resolveLocalPathFromMarkdownUrl(url: string): string | null {return null;}+// Media paths go to the native file reader, so a URL that merely ends in+// .mp4 is not enough: it must be a local path, or home-relative (`~/clip.mp4`,+// which the reader expands).+function isLocalMediaPath(url: string): boolean {+ if (!isMediaFilePath(url)) return false;+ return /^~[\\/]/.test(unwrapMarkdownUrl(url)) || resolveLocalPathFromMarkdownUrl(url) !== null;+}+export function createScreenpipeUrlTransform(allowedHosts: readonly string[]) {const allowed = new Set(allowedHosts);@@ -166,7 +174,7 @@ export function createMediaAwareMarkdownComponents(return {...base,a({ href, children, ...props }) {- if (href && isMediaFilePath(href)) {+ if (href && isLocalMediaPath(href)) {return <MediaComponent filePath={href} className="my-2" />;}@@ -177,10 +185,10 @@ export function createMediaAwareMarkdownComponents(return <a href={href} {...props}>{children}</a>;},- img({ src, alt, ...props }) {+ img({ src, alt }) {if (!src) return null;- if (isMediaFilePath(src)) {+ if (isLocalMediaPath(src)) {return <MediaComponent filePath={src} className="my-2" />;}@@ -195,25 +203,22 @@ export function createMediaAwareMarkdownComponents();}+ // Markdown here is often written by an AI or a pipe working from+ // captured screens, pages and files, so outside content can steer its+ // image URLs. A remote image loads as soon as it renders and hands its+ // URL to that server, so only local files render as images; anything+ // else shows its alt text (same rule as announcement-body.tsx). A+ // caller's img never receives the src, so it cannot load it either.const CustomImage = base.img;if (CustomImage) {- return <CustomImage src={src} alt={alt} {...props} />;+ return <CustomImage alt={alt} />;}- return (- // eslint-disable-next-line @next/next/no-img-element- <img- src={src}- alt={alt || ""}- className="max-w-full h-auto rounded-md my-2 border border-border"- loading="lazy"- {...props}- />- );+ return alt ? <span>{alt}</span> : null;},code({ className, children, ...props }) {const content = String(children).replace(/\n$/, "");- if (isMediaFilePath(content.trim())) {+ if (isLocalMediaPath(content.trim())) {return <MediaComponent filePath={content.trim()} className="my-2" />;}diff --git a/apps/screenpipe-app-tauri/components/notification-bell.test.tsx b/apps/screenpipe-app-tauri/components/notification-bell.test.tsxindex 5ca53c501..e152d46d4 100644--- a/apps/screenpipe-app-tauri/components/notification-bell.test.tsx+++ b/apps/screenpipe-app-tauri/components/notification-bell.test.tsx@@ -71,6 +71,29 @@ describe("NotificationInboxPanel", () => {expect(screen.getByTestId("notification-bell-view-all")).toHaveTextContent("0");});+ it("shows remote images in bodies as alt text in both row views", async () => {+ // Pipes write these bodies; a rendered remote image would send its URL+ // to that server the moment the inbox opens.+ const remoteImage = {+ ...entries[0],+ id: "remote-image",+ body: "chart below ",+ };+ appServerFetch.mockImplementation(() =>+ Promise.resolve({ ok: true, json: () => Promise.resolve([remoteImage]) }),+ );+ const { container } = render(<NotificationInboxPanel />);++ const row = await screen.findByTestId("notification-bell-item-remote-image");+ expect(row).toHaveTextContent("chart below remote chart");+ expect(container.querySelector("img")).toBeNull();++ fireEvent.click(row);+ const expanded = await screen.findByTestId("notification-bell-expanded-remote-image");+ expect(expanded).toHaveTextContent("chart below remote chart");+ expect(container.querySelector("img")).toBeNull();+ });+it("hides clear all only when there is nothing to clear", async () => {appServerFetch.mockImplementation(() =>Promise.resolve({ ok: true, json: () => Promise.resolve([]) }),diff --git a/apps/screenpipe-app-tauri/components/notification-bell.tsx b/apps/screenpipe-app-tauri/components/notification-bell.tsxindex 781d01570..e710d69c3 100644--- a/apps/screenpipe-app-tauri/components/notification-bell.tsx+++ b/apps/screenpipe-app-tauri/components/notification-bell.tsx@@ -461,6 +461,9 @@ export function NotificationInboxPanel({{children}</a>),+ // Bodies come from pipes; a remote image would load on+ // render and send its URL out, so show the alt text.+ img: ({ alt }) => (alt ? <span>{alt}</span> : null),}}>{entry.body}@@ -537,6 +540,7 @@ export function NotificationInboxPanel({{children}</a>),+ img: ({ alt }) => (alt ? <span>{alt}</span> : null),}}>{entry.body}</ReactMarkdown></div>diff --git a/apps/screenpipe-app-tauri/lib/__tests__/markdown-remote-images.test.ts b/apps/screenpipe-app-tauri/lib/__tests__/markdown-remote-images.test.tsnew file mode 100644index 000000000..1f2007892--- /dev/null+++ b/apps/screenpipe-app-tauri/lib/__tests__/markdown-remote-images.test.ts@@ -0,0 +1,68 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { describe, expect, it } from "vitest";+import { readFileSync, readdirSync } from "node:fs";+import { relative, resolve } from "node:path";++// Markdown in this app is mostly written by an AI or a pipe working from+// captured screens, pages and files, so outside content can steer its image+// URLs. A remote image loads the moment it renders and sends its URL to that+// server, with no click. MemoizedReactMarkdown (components/markdown.tsx)+// renders only local files as images; each other direct react-markdown+// renderer below was checked to override or exclude `img`.+const REVIEWED_RENDERERS = [+ "app/notification-panel/page.tsx", // img shows alt text+ "components/announcement-body.tsx", // img shows alt text+ "components/markdown.tsx", // MemoizedReactMarkdown: local files only+ "components/notification-bell.tsx", // img shows alt text+ "components/settings/live-view-card.tsx", // allowedElements excludes img+];++// Runtime imports only; `import type` cannot render anything.+const RUNTIME_IMPORT =+ /\b(?:import|export)\s+(?!type\b)[^;'"]*\bfrom\s*["']react-markdown["']|\bimport\(\s*["']react-markdown["']/;++function frontendSourceFiles(directory: string): string[] {+ return readdirSync(directory, { withFileTypes: true }).flatMap((dirent) => {+ const path = resolve(directory, dirent.name);+ if (dirent.isDirectory()) {+ return dirent.name === "__tests__" || dirent.name === "node_modules"+ ? []+ : frontendSourceFiles(path);+ }+ if (!/\.(?:ts|tsx)$/.test(dirent.name)) return [];+ if (/\.(?:spec|test)\./.test(dirent.name)) return [];+ return [path];+ });+}++describe("markdown remote images", () => {+ it("renders markdown only through renderers reviewed for remote images", () => {+ const root = process.cwd();+ const renderers = ["app", "components", "lib"]+ .flatMap((dir) => frontendSourceFiles(resolve(root, dir)))+ .filter((file) => RUNTIME_IMPORT.test(readFileSync(file, "utf8")))+ .map((file) => relative(root, file).replaceAll("\\", "/"))+ .sort();++ expect(+ renderers,+ "Render markdown with MemoizedReactMarkdown from components/markdown.tsx. " ++ "If a file must use react-markdown directly, give it an `img` component " ++ "that never loads a remote URL, then add it to REVIEWED_RENDERERS.",+ ).toEqual(REVIEWED_RENDERERS);+ });++ it.each([+ [`import ReactMarkdown from "react-markdown";`, true],+ [`import ReactMarkdown, { defaultUrlTransform, Options } from 'react-markdown'`, true],+ [`import {\n MarkdownHooks,\n} from "react-markdown";`, true],+ [`export { default } from "react-markdown";`, true],+ [`const Markdown = await import("react-markdown");`, true],+ [`import type { Options } from "react-markdown";`, false],+ [`import remarkGfm from "remark-gfm";`, false],+ ])("detects runtime react-markdown imports in %s", (source, expected) => {+ expect(RUNTIME_IMPORT.test(source)).toBe(expected);+ });+});diff --git a/evals/coding-agent/graders/chat-html-outcomes.fixture.tsx.txt b/evals/coding-agent/graders/chat-html-outcomes.fixture.tsx.txtindex bf7a72c62..fe1a77072 100644--- a/evals/coding-agent/graders/chat-html-outcomes.fixture.tsx.txt+++ b/evals/coding-agent/graders/chat-html-outcomes.fixture.tsx.txt@@ -95,11 +95,12 @@ describe("HTML outcome probe", () => { afterEach(() => { vi.clearAllMocks(); });const container = renderHtml(`<a href="javascript:alert(1)">click</a> <img src="https://example.com/x.png" longdesc="javascript:alert(1)">`,);+ // The message must render before the absence checks mean anything.+ expect(container).toHaveTextContent("click");expect(container.querySelector("a[href^='javascript']")).toBeNull();- const image = container.querySelector("img");- expect(image).not.toBeNull();- expect(image?.getAttribute("src")).toBe("https://example.com/x.png");- expect(image?.getAttribute("longdesc") ?? "").not.toMatch(/^\s*javascript:/i);+ // Whether the remote image renders is a later product rule+ // (app-markdown-remote-images); wherever it does, no javascript: URL may.+ expect(container.innerHTML).not.toMatch(/javascript:/i);});it("keeps safe formatting and collapsible details", () => {