Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Merge upstream/main into fix/markdown-remote-images

Manas Raghuwanshie13354eded6b
diff --cc apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx
index 7f52aae04,aa447709d..b47c7840b
--- a/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx
+++ b/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx
@@@ -94,85 -90,10 +94,81 @@@ describe("MemoizedReactMarkdown local i
expect(container.querySelector("img")).toBeNull();
});
- it("leaves remote https images on their original src", () => {
+ // A remote image loads the moment it renders and sends its URL to that
+ // server, so markdown shows the alt text instead of fetching anything.
+ it.each([
+ ["https", "![remote chart](https://example.com/pipe.png?d=secret)"],
+ ["http", "![remote chart](http://example.com/pipe.png)"],
+ ["protocol-relative", "![remote chart](//example.com/pipe.png)"],
+ ["reference-style", "![remote chart][r]\n\n[r]: https://example.com/pipe.png"],
+ ["relative", "![remote chart](pipe.png)"],
+ ])("shows %s images as alt text without loading them", (_name, markdown) => {
+ const { container } = render(
+ <MemoizedReactMarkdown urlTransform={chatUrlTransform}>
+ {markdown}
+ </MemoizedReactMarkdown>,
+ );
+
+ expect(screen.getByText("remote chart")).toBeInTheDocument();
+ expect(container.querySelector("img")).toBeNull();
+ expect(getMediaFileMock).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["javascript:", "![remote chart](javascript:alert(1))"],
+ ["data:", "![remote chart](data:image/png;base64,iVBORw0KGgo=)"],
+ ["blob:", "![remote chart](blob:https://example.com/0b1c)"],
+ ])("keeps the alt text of a %s image whose src the url transform removes", (_name, markdown) => {
+ const { container } = render(
+ <MemoizedReactMarkdown urlTransform={chatUrlTransform}>
+ {`before ${markdown} after`}
+ </MemoizedReactMarkdown>,
+ );
+
+ expect(container).toHaveTextContent("before remote chart after");
+ expect(container.querySelector("img")).toBeNull();
+ });
+
+ it("does not load remote images without a url transform or alt text", () => {
+ // Daily summaries and meeting notes use react-markdown's default transform.
+ const { container } = render(
+ <MemoizedReactMarkdown>
+ {"before ![](https://example.com/pipe.png) after ![remote chart](https://example.com/b.png)"}
+ </MemoizedReactMarkdown>,
+ );
+
+ expect(container).toHaveTextContent("before after remote chart");
+ expect(container.querySelector("img")).toBeNull();
+ });
+
+ it.each([
+ ["remote media image", "![clip](https://example.com/a.mp4)", () => screen.getByText("clip")],
+ [
+ "remote media link",
+ "[clip](https://example.com/a.mp4)",
+ () => expect(screen.getByRole("link", { name: "clip" })).toHaveAttribute("href", "https://example.com/a.mp4"),
+ ],
+ ["remote media code", "`https://example.com/a.mp4`", () => screen.getByText("https://example.com/a.mp4")],
+ ["network-share media image", "![clip](//host/share/a.mp4)", () => screen.getByText("clip")],
+ ["network-share media code", "`\\\\host\\share\\a.mp4`", () => screen.getByText("\\\\host\\share\\a.mp4")],
+ // `~/` + a share: the native reader's home join would yield the share.
+ [
+ "home-relative network-share code",
+ "`~/\\\\host\\share\\a.mp4`",
+ () => screen.getByText("~/\\\\host\\share\\a.mp4"),
+ ],
+ // The reader extracts a path from the string; here that is the share.
- [
- "network share after a quote in a link",
- '[clip](</tmp/x"//host/share/a.mp4>)',
- () => screen.getByRole("link", { name: "clip" }),
- ],
++ ["network share after a quote in a link", '[clip](</tmp/x"//host/share/a.mp4>)', () => screen.getByText("clip")],
+ ["network share after a quote in an image", '![clip](</tmp/x"//host/share/a.mp4>)', () => screen.getByText("clip")],
+ [
+ "network share on a second code line",
+ "```\n/tmp/x\n//host/share/a.mp4\n```",
+ () => expect(document.querySelector("code")).toHaveTextContent("//host/share/a.mp4"),
+ ],
+ ])("keeps %s away from the native media reader", (_name, markdown, expectOrdinaryForm) => {
render(
<MemoizedReactMarkdown urlTransform={chatUrlTransform}>
- {"![remote](https://example.com/pipe.png)"}
+ {markdown}
</MemoizedReactMarkdown>,
);
diff --cc apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx
index f47eefe21,ce78f7934..a33576400
--- a/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx
+++ b/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx
@@@ -428,49 -512,10 +512,49 @@@ describe("MarkdownBlock", () =>
it("neutralizes javascript: URLs in raw HTML attributes", () => {
const container = renderHtml(
- `<a href="javascript:alert(1)">click</a> <img src="https://example.com/x.png" longdesc="javascript:alert(1)">`,
+ `<a href="javascript:alert(1)">click</a> <img src="https://example.com/x.png" alt="remote chart" longdesc="javascript:alert(1)">\n\n<blockquote cite="javascript:alert(1)">quoted</blockquote>`,
);
+ expect(screen.getByText("click")).toBeInTheDocument();
expect(container.querySelector("a[href^='javascript']")).toBeNull();
- expect(container.querySelector("img")?.getAttribute("longdesc")).toBeNull();
+ expect(container.querySelector("img")).toBeNull();
+ expect(container).toHaveTextContent("remote chart");
+ const quote = container.querySelector("blockquote");
+ expect(quote).toHaveTextContent("quoted");
+ expect(quote?.getAttribute("cite") ?? "").not.toMatch(/javascript:/i);
+ expect(container.innerHTML).not.toMatch(/javascript:/i);
+ });
+
+ // A remote image loads as soon as it renders and sends its URL (and any
+ // data the AI was steered into putting there) to that server.
+ it.each([
+ ["markdown image", `![remote chart](https://example.com/x.png?d=secret)`],
+ ["raw img", `<img src="https://example.com/x.png?d=secret" alt="remote chart">`],
+ ["protocol-relative raw img", `<img src="//example.com/x.png" alt="remote chart">`],
+ ])("shows a remote %s as alt text without loading it", (_name, html) => {
+ const container = renderHtml(`before\n\n${html}\n\nafter`);
+ expect(container).toHaveTextContent("before");
+ expect(container).toHaveTextContent("remote chart");
+ expect(container).toHaveTextContent("after");
+ expect(container.querySelector("img")).toBeNull();
+ });
+
+ it("drops picture/source so a srcset cannot replace a local image", async () => {
- getMediaFileMock.mockResolvedValue({
- data: "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",
- mimeType: "image/gif",
++ getMediaFileMock.mockResolvedValueOnce({
++ status: "ok",
++ data: { data: "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", mimeType: "image/gif" },
+ });
+ URL.createObjectURL = vi.fn(() => "blob:local-picture");
+ URL.revokeObjectURL = vi.fn();
+
+ const container = renderHtml(
+ `before\n\n<picture><source srcset="https://example.com/x.png?d=secret"><img src="/tmp/local.gif" alt="pic"></picture>\n\nafter`,
+ );
+
+ expect(await screen.findByAltText("pic")).toHaveAttribute("src", "blob:local-picture");
+ expect(getMediaFileMock).toHaveBeenCalledWith("/tmp/local.gif");
+ expect(container.querySelector("picture, source, [srcset]")).toBeNull();
+ expect(container).toHaveTextContent("before");
+ expect(container).toHaveTextContent("after");
});
it("keeps safe formatting and collapsible details", () => {
diff --cc apps/screenpipe-app-tauri/components/markdown.tsx
index 18a94b8e1,5e7deb27a..e5fbcf956
--- a/apps/screenpipe-app-tauri/components/markdown.tsx
+++ b/apps/screenpipe-app-tauri/components/markdown.tsx
@@@ -6,9 -5,15 +5,16 @@@ import ReactMarkdown, { defaultUrlTrans
import { commands } from "@/lib/utils/tauri";
import { MediaComponent } from "@/components/rewind/media";
import { LocalMarkdownImage } from "@/components/markdown/local-markdown-image";
+import { ImageAltText } from "@/components/markdown/image-alt-text";
import { imageMimeFromName } from "@/components/meeting-notes/image-utils";
- import { isMediaFilePath, normalizeLocalMediaMarkdown, normalizeMediaFilePath } from "@/lib/utils/media-file-path";
+ import {
+ decodeLinkAddress,
+ isMediaAddress,
+ isMediaFilePath,
+ normalizeLocalMediaMarkdown,
+ normalizeMediaFilePath,
+ runsPastLinkEnd,
+ } from "@/lib/utils/media-file-path";
function unwrapMarkdownUrl(url: string): string {
const trimmed = url.trim();
@@@ -56,17 -61,6 +62,18 @@@ export function resolveLocalPathFromMar
return null;
}
- // Media paths go to the native file reader, so a URL that merely ends in .mp4
- // is not enough. MediaComponent reads normalizeMediaFilePath(url), which
- // decodes the URL and can pull a path out of a longer string, so check that
- // exact path: it must be absolute, a drive path, or home-relative
- // (`~/clip.mp4`, which the reader expands). A second leading slash or
- // backslash is a network share on Windows.
++// Media paths go to the native file reader. MediaComponent reads
++// normalizeMediaFilePath(url), so check that exact path: it must be absolute,
++// a drive path, or home-relative (`~/clip.mp4`, which the reader expands), and
++// not a network share: on Windows a share path (`\\host\share`, `//host/share`,
++// `/\host\share`, `~/\\host\share`) makes the reader contact that host.
+function isLocalMediaPath(url: string): boolean {
- const path = normalizeMediaFilePath(url);
- return isMediaFilePath(path) && /^(?:\/|~[\\/]|[A-Za-z]:[\\/])(?![\\/])/.test(path);
++ return (
++ isMediaFilePath(url) &&
++ /^(?:\/|~[\\/]|[A-Za-z]:[\\/])(?![\\/])/.test(normalizeMediaFilePath(url))
++ );
+}
+
export function createScreenpipeUrlTransform(allowedHosts: readonly string[]) {
const allowed = new Set(allowedHosts);
@@@ -178,23 -215,33 +228,31 @@@ export function createMediaAwareMarkdow
return {
...base,
a({ href, children, ...props }) {
- if (href && isMediaFilePath(href)) {
+ if (href && isLocalMediaPath(href)) {
- return <MediaComponent filePath={href} className="my-2" />;
+ return (
+ <MediaComponent
+ filePath={href}
+ className="my-2"
+ fallback={mediaLinkAsText(href, children)}
+ />
+ );
}
-
- const CustomAnchor = base.a;
- if (CustomAnchor) {
- return <CustomAnchor href={href} {...props}>{children}</CustomAnchor>;
+ if (href && isMediaAddress(href) && !WEB_ADDRESS.test(href)) {
+ return mediaLinkAsText(href, children);
}
-
- return <a href={href} {...props}>{children}</a>;
+ return link(href, children, props);
},
- img({ src, alt, ...props }) {
- if (!src) return null;
-
+ img({ src, alt }) {
+ // An <img> can't show audio or video, so a media address plays, opens
+ // as a web link, or reads as text.
- if (isMediaFilePath(src)) {
+ if (src && isLocalMediaPath(src)) {
- return <MediaComponent filePath={src} className="my-2" />;
+ return <MediaComponent filePath={src} className="my-2" fallback={mediaLinkAsText(src, alt)} />;
+ }
- if (isMediaAddress(src)) {
++ if (src && isMediaAddress(src)) {
+ return WEB_ADDRESS.test(src) ? link(src, alt || src) : mediaLinkAsText(src, alt);
}
- const localPath = resolveLocalPathFromMarkdownUrl(src);
+ const localPath = src ? resolveLocalPathFromMarkdownUrl(src) : null;
if (localPath && imageMimeFromName(localPath)) {
return (
<LocalMarkdownImage
@@@ -205,28 -252,35 +263,29 @@@
);
}
+ // Only local files render as images; anything else, including a src the
+ // url transform blanked, shows its alt text (see ImageAltText). A
+ // caller's img never receives the src, so it cannot load it either.
const CustomImage = base.img;
if (CustomImage) {
- return <CustomImage src={src} alt={alt} {...props} />;
+ return <CustomImage alt={alt} />;
}
- return (
- // eslint-disable-next-line @next/next/no-img-element
- <img
- src={src}
- alt={alt || ""}
- className="max-w-full h-auto rounded-md my-2 border border-border"
- loading="lazy"
- {...props}
- />
- );
+ return <ImageAltText alt={alt} />;
},
code({ className, children, ...props }) {
- const content = String(children).replace(/\n$/, "");
- if (isLocalMediaPath(content.trim())) {
- return <MediaComponent filePath={content.trim()} className="my-2" />;
- }
-
const CustomCode = base.code;
- if (CustomCode) {
- return <CustomCode className={className} {...props}>{children}</CustomCode>;
+ const codeText = CustomCode ? (
+ <CustomCode className={className} {...props}>{children}</CustomCode>
+ ) : (
+ <code className={className} {...props}>{children}</code>
+ );
+
+ const content = String(children).replace(/\n$/, "").trim();
- if (isMediaFilePath(content)) {
++ if (isLocalMediaPath(content)) {
+ return <MediaComponent filePath={content} className="my-2" fallback={codeText} />;
}
-
- return <code className={className} {...props}>{children}</code>;
+ return codeText;
},
};
}