YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Merge upstream/main into fix/markdown-remote-images
Manas Raghuwanshie13354eded6b
diff --cc apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsxindex 7f52aae04,aa447709d..b47c7840b--- a/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx+++ b/apps/screenpipe-app-tauri/components/__tests__/markdown-local-image.test.tsx@@@ -94,85 -90,10 +94,81 @@@ describe("MemoizedReactMarkdown local iexpect(container.querySelector("img")).toBeNull();});- it("leaves remote https images on their original src", () => {+ // A remote image loads the moment it renders and sends its URL to that+ // server, so markdown shows the alt text instead of fetching anything.+ it.each([+ ["https", ""],+ ["http", ""],+ ["protocol-relative", ""],+ ["reference-style", "![remote chart][r]\n\n[r]: https://example.com/pipe.png"],+ ["relative", ""],+ ])("shows %s images as alt text without loading them", (_name, markdown) => {+ const { container } = render(+ <MemoizedReactMarkdown urlTransform={chatUrlTransform}>+ {markdown}+ </MemoizedReactMarkdown>,+ );++ expect(screen.getByText("remote chart")).toBeInTheDocument();+ expect(container.querySelector("img")).toBeNull();+ expect(getMediaFileMock).not.toHaveBeenCalled();+ });++ it.each([+ ["javascript:", ")"],+ ["data:", ""],+ ["blob:", ""],+ ])("keeps the alt text of a %s image whose src the url transform removes", (_name, markdown) => {+ const { container } = render(+ <MemoizedReactMarkdown urlTransform={chatUrlTransform}>+ {`before ${markdown} after`}+ </MemoizedReactMarkdown>,+ );++ expect(container).toHaveTextContent("before remote chart after");+ expect(container.querySelector("img")).toBeNull();+ });++ it("does not load remote images without a url transform or alt text", () => {+ // Daily summaries and meeting notes use react-markdown's default transform.+ const { container } = render(+ <MemoizedReactMarkdown>+ {"before  after "}+ </MemoizedReactMarkdown>,+ );++ expect(container).toHaveTextContent("before after remote chart");+ expect(container.querySelector("img")).toBeNull();+ });++ it.each([+ ["remote media image", "", () => screen.getByText("clip")],+ [+ "remote media link",+ "[clip](https://example.com/a.mp4)",+ () => expect(screen.getByRole("link", { name: "clip" })).toHaveAttribute("href", "https://example.com/a.mp4"),+ ],+ ["remote media code", "`https://example.com/a.mp4`", () => screen.getByText("https://example.com/a.mp4")],+ ["network-share media image", "", () => screen.getByText("clip")],+ ["network-share media code", "`\\\\host\\share\\a.mp4`", () => screen.getByText("\\\\host\\share\\a.mp4")],+ // `~/` + a share: the native reader's home join would yield the share.+ [+ "home-relative network-share code",+ "`~/\\\\host\\share\\a.mp4`",+ () => screen.getByText("~/\\\\host\\share\\a.mp4"),+ ],+ // The reader extracts a path from the string; here that is the share.- [- "network share after a quote in a link",- '[clip](</tmp/x"//host/share/a.mp4>)',- () => screen.getByRole("link", { name: "clip" }),- ],++ ["network share after a quote in a link", '[clip](</tmp/x"//host/share/a.mp4>)', () => screen.getByText("clip")],+ ["network share after a quote in an image", '', () => screen.getByText("clip")],+ [+ "network share on a second code line",+ "```\n/tmp/x\n//host/share/a.mp4\n```",+ () => expect(document.querySelector("code")).toHaveTextContent("//host/share/a.mp4"),+ ],+ ])("keeps %s away from the native media reader", (_name, markdown, expectOrdinaryForm) => {render(<MemoizedReactMarkdown urlTransform={chatUrlTransform}>- {""}+ {markdown}</MemoizedReactMarkdown>,);diff --cc apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsxindex f47eefe21,ce78f7934..a33576400--- a/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx+++ b/apps/screenpipe-app-tauri/components/chat/markdown-block.test.tsx@@@ -428,49 -512,10 +512,49 @@@ describe("MarkdownBlock", () =>it("neutralizes javascript: URLs in raw HTML attributes", () => {const container = renderHtml(- `<a href="javascript:alert(1)">click</a> <img src="https://example.com/x.png" longdesc="javascript:alert(1)">`,+ `<a href="javascript:alert(1)">click</a> <img src="https://example.com/x.png" alt="remote chart" longdesc="javascript:alert(1)">\n\n<blockquote cite="javascript:alert(1)">quoted</blockquote>`,);+ expect(screen.getByText("click")).toBeInTheDocument();expect(container.querySelector("a[href^='javascript']")).toBeNull();- expect(container.querySelector("img")?.getAttribute("longdesc")).toBeNull();+ expect(container.querySelector("img")).toBeNull();+ expect(container).toHaveTextContent("remote chart");+ const quote = container.querySelector("blockquote");+ expect(quote).toHaveTextContent("quoted");+ expect(quote?.getAttribute("cite") ?? "").not.toMatch(/javascript:/i);+ expect(container.innerHTML).not.toMatch(/javascript:/i);+ });++ // A remote image loads as soon as it renders and sends its URL (and any+ // data the AI was steered into putting there) to that server.+ it.each([+ ["markdown image", ``],+ ["raw img", `<img src="https://example.com/x.png?d=secret" alt="remote chart">`],+ ["protocol-relative raw img", `<img src="//example.com/x.png" alt="remote chart">`],+ ])("shows a remote %s as alt text without loading it", (_name, html) => {+ const container = renderHtml(`before\n\n${html}\n\nafter`);+ expect(container).toHaveTextContent("before");+ expect(container).toHaveTextContent("remote chart");+ expect(container).toHaveTextContent("after");+ expect(container.querySelector("img")).toBeNull();+ });++ it("drops picture/source so a srcset cannot replace a local image", async () => {- getMediaFileMock.mockResolvedValue({- data: "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",- mimeType: "image/gif",++ getMediaFileMock.mockResolvedValueOnce({++ status: "ok",++ data: { data: "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", mimeType: "image/gif" },+ });+ URL.createObjectURL = vi.fn(() => "blob:local-picture");+ URL.revokeObjectURL = vi.fn();++ const container = renderHtml(+ `before\n\n<picture><source srcset="https://example.com/x.png?d=secret"><img src="/tmp/local.gif" alt="pic"></picture>\n\nafter`,+ );++ expect(await screen.findByAltText("pic")).toHaveAttribute("src", "blob:local-picture");+ expect(getMediaFileMock).toHaveBeenCalledWith("/tmp/local.gif");+ expect(container.querySelector("picture, source, [srcset]")).toBeNull();+ expect(container).toHaveTextContent("before");+ expect(container).toHaveTextContent("after");});it("keeps safe formatting and collapsible details", () => {diff --cc apps/screenpipe-app-tauri/components/markdown.tsxindex 18a94b8e1,5e7deb27a..e5fbcf956--- a/apps/screenpipe-app-tauri/components/markdown.tsx+++ b/apps/screenpipe-app-tauri/components/markdown.tsx@@@ -6,9 -5,15 +5,16 @@@ import ReactMarkdown, { defaultUrlTransimport { commands } from "@/lib/utils/tauri";import { MediaComponent } from "@/components/rewind/media";import { LocalMarkdownImage } from "@/components/markdown/local-markdown-image";+import { ImageAltText } from "@/components/markdown/image-alt-text";import { imageMimeFromName } from "@/components/meeting-notes/image-utils";- import { isMediaFilePath, normalizeLocalMediaMarkdown, normalizeMediaFilePath } from "@/lib/utils/media-file-path";+ import {+ decodeLinkAddress,+ isMediaAddress,+ isMediaFilePath,+ normalizeLocalMediaMarkdown,+ normalizeMediaFilePath,+ runsPastLinkEnd,+ } from "@/lib/utils/media-file-path";function unwrapMarkdownUrl(url: string): string {const trimmed = url.trim();@@@ -56,17 -61,6 +62,18 @@@ export function resolveLocalPathFromMarreturn null;}- // Media paths go to the native file reader, so a URL that merely ends in .mp4- // is not enough. MediaComponent reads normalizeMediaFilePath(url), which- // decodes the URL and can pull a path out of a longer string, so check that- // exact path: it must be absolute, a drive path, or home-relative- // (`~/clip.mp4`, which the reader expands). A second leading slash or- // backslash is a network share on Windows.++// Media paths go to the native file reader. MediaComponent reads++// normalizeMediaFilePath(url), so check that exact path: it must be absolute,++// a drive path, or home-relative (`~/clip.mp4`, which the reader expands), and++// not a network share: on Windows a share path (`\\host\share`, `//host/share`,++// `/\host\share`, `~/\\host\share`) makes the reader contact that host.+function isLocalMediaPath(url: string): boolean {- const path = normalizeMediaFilePath(url);- return isMediaFilePath(path) && /^(?:\/|~[\\/]|[A-Za-z]:[\\/])(?![\\/])/.test(path);++ return (++ isMediaFilePath(url) &&++ /^(?:\/|~[\\/]|[A-Za-z]:[\\/])(?![\\/])/.test(normalizeMediaFilePath(url))++ );+}+export function createScreenpipeUrlTransform(allowedHosts: readonly string[]) {const allowed = new Set(allowedHosts);@@@ -178,23 -215,33 +228,31 @@@ export function createMediaAwareMarkdowreturn {...base,a({ href, children, ...props }) {- if (href && isMediaFilePath(href)) {+ if (href && isLocalMediaPath(href)) {- return <MediaComponent filePath={href} className="my-2" />;+ return (+ <MediaComponent+ filePath={href}+ className="my-2"+ fallback={mediaLinkAsText(href, children)}+ />+ );}-- const CustomAnchor = base.a;- if (CustomAnchor) {- return <CustomAnchor href={href} {...props}>{children}</CustomAnchor>;+ if (href && isMediaAddress(href) && !WEB_ADDRESS.test(href)) {+ return mediaLinkAsText(href, children);}-- return <a href={href} {...props}>{children}</a>;+ return link(href, children, props);},- img({ src, alt, ...props }) {- if (!src) return null;-+ img({ src, alt }) {+ // An <img> can't show audio or video, so a media address plays, opens+ // as a web link, or reads as text.- if (isMediaFilePath(src)) {+ if (src && isLocalMediaPath(src)) {- return <MediaComponent filePath={src} className="my-2" />;+ return <MediaComponent filePath={src} className="my-2" fallback={mediaLinkAsText(src, alt)} />;+ }- if (isMediaAddress(src)) {++ if (src && isMediaAddress(src)) {+ return WEB_ADDRESS.test(src) ? link(src, alt || src) : mediaLinkAsText(src, alt);}- const localPath = resolveLocalPathFromMarkdownUrl(src);+ const localPath = src ? resolveLocalPathFromMarkdownUrl(src) : null;if (localPath && imageMimeFromName(localPath)) {return (<LocalMarkdownImage@@@ -205,28 -252,35 +263,29 @@@);}+ // Only local files render as images; anything else, including a src the+ // url transform blanked, shows its alt text (see ImageAltText). A+ // caller's img never receives the src, so it cannot load it either.const CustomImage = base.img;if (CustomImage) {- return <CustomImage src={src} alt={alt} {...props} />;+ return <CustomImage alt={alt} />;}- return (- // eslint-disable-next-line @next/next/no-img-element- <img- src={src}- alt={alt || ""}- className="max-w-full h-auto rounded-md my-2 border border-border"- loading="lazy"- {...props}- />- );+ return <ImageAltText alt={alt} />;},code({ className, children, ...props }) {- const content = String(children).replace(/\n$/, "");- if (isLocalMediaPath(content.trim())) {- return <MediaComponent filePath={content.trim()} className="my-2" />;- }-const CustomCode = base.code;- if (CustomCode) {- return <CustomCode className={className} {...props}>{children}</CustomCode>;+ const codeText = CustomCode ? (+ <CustomCode className={className} {...props}>{children}</CustomCode>+ ) : (+ <code className={className} {...props}>{children}</code>+ );++ const content = String(children).replace(/\n$/, "").trim();- if (isMediaFilePath(content)) {++ if (isLocalMediaPath(content)) {+ return <MediaComponent filePath={content} className="my-2" fallback={codeText} />;}-- return <code className={className} {...props}>{children}</code>;+ return codeText;},};}