YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover remote markdown image outcomes
Manas Raghuwanshiae8c3caee84f
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex fc7aa98e4..7a2b45b02 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1274,6 +1274,24 @@ output, one-role-only repair, dropped safe HTML, styles, broken fragments andmissing-source setup failure. These are corpus/grader checks, not agent trials,a full browser exploit defense or proof of execution isolation.+### Remote Markdown images++`app-markdown-remote-images` renders the actual chat MarkdownBlock (both roles)+and the shared MemoizedReactMarkdown with a synthetic native media port. Forty-six+outcomes check that remote, protocol-relative, relative, reference-style, raw+`<img>` and `<picture><source srcset>` images, and remote or network-share media,+create no element that loads them and never reach the media reader, while their+alt text, link or code stays visible. Local absolute, file-URL, Windows and+home-relative files still render through the native reader. The parent fails+thirty-nine outcomes and preserves seven; the fix and current source pass all.++Run `bun test evals/coding-agent/calibrate-markdown-remote-images.test.js`.+Controls include parent/reference/current, an equivalent alt-text element, a+restored remote `<img>` fallback, re-allowed picture/source, an extension-only+media check, network shares treated as local, blanket image removal, unused+correct source and missing-source setup failure. Webview CSP, mermaid, the note+editor, native file policy and execution isolation are outside this evidence.+`ai-gateway-trusted-runner-admission` runs the actual HTTP entrypoint, authentication,Free-plan gates, chat handler and provider adapter with synthetic external ports.Fourteen outcomes cover successful machine and paid-human replies, forged ordiff --git a/evals/coding-agent/calibrate-markdown-remote-images.test.js b/evals/coding-agent/calibrate-markdown-remote-images.test.jsnew file mode 100644index 000000000..874d491e3--- /dev/null+++ b/evals/coding-agent/calibrate-markdown-remote-images.test.js@@ -0,0 +1,54 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-markdown-remote-images');+const markdown = `${app}/components/markdown.tsx`, block = `${app}/components/chat/markdown-block.tsx`;+const root = mkdtempSync(join(tmpdir(), 'markdown-remote-images-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ for (const link of item.grader_dependency_links) { const dest = join(cwd, link.destination_path); mkdirSync(dirname(dest),{recursive:true}); symlinkSync(join(repo,link.source_path),dest,'dir'); }+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:120000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+// Behavior failures surface as chai AssertionErrors or jest-dom matcher errors.+const BEHAVIOR_FAILURE = /AssertionError|Error: expect\(/;+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('46 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toMatch(BEHAVIOR_FAILURE); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails thirty-nine remote outcomes and preserves seven local files', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stdout).toContain('39 failed | 7 passed'); }, 120000);+test('historical reference passes every outcome', () => passes(grade('reference')), 120000);+test('current caller passes every outcome', () => passes(grade('current', 'HEAD')), 120000);+test('equivalent alt-text element passes', () => passes(grade('equivalent', item.oracle_ref, cwd => replace(cwd, markdown, 'return alt ? <span>{alt}</span> : null;', 'return alt ? <em>{alt}</em> : null;'))), 120000);+test('plain remote img fallback fails', () => fails(grade('img-fallback', item.oracle_ref, cwd => replace(cwd, markdown, 'return alt ? <span>{alt}</span> : null;', 'return <img src={src} alt={alt || ""} />;'))), 120000);+test('re-allowed picture and source fail', () => fails(grade('picture', item.oracle_ref, cwd => replace(cwd, block, '...(defaultSchema.tagNames ?? []).filter((tag) => tag !== "picture" && tag !== "source"),', '...(defaultSchema.tagNames ?? []),'))), 120000);+test('extension-only media gate fails', () => fails(grade('media-gate', item.oracle_ref, cwd => replace(cwd, markdown, 'return /^~[\\\\/]/.test(unwrapMarkdownUrl(url)) || resolveLocalPathFromMarkdownUrl(url) !== null;', 'return true;'))), 120000);+test('network-share paths treated as local fail', () => fails(grade('network-share', item.oracle_ref, cwd => replace(cwd, markdown, 'if (/^\\/(?![\\\\/])/.test(candidate)) {', 'if (candidate.startsWith("/")) {'))), 120000);+test('blanket image removal fails preserved local files', () => fails(grade('no-images', item.oracle_ref, cwd => replace(cwd, markdown, ' img({ src, alt }) {\n if (!src) return null;', ' img({ src, alt }) {\n if (src || !src) return null;'))), 120000);+test('unused correct renderer cannot hide broken active caller', () => fails(grade('unused', item.oracle_ref, cwd => {+ writeFileSync(join(cwd, `${app}/unused-correct-markdown.tsx`), readFileSync(join(cwd, markdown)));+ writeFileSync(join(cwd, markdown), execFileSync('git', ['show', `${item.base_ref}:${markdown}`], {cwd: repo}));+})), 120000);+test('missing renderer is setup failure', () => { const r = grade('missing', item.oracle_ref, cwd => rmSync(join(cwd, markdown))); expect(r.status).toBe(1); expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/); expect(r.stderr).not.toMatch(BEHAVIOR_FAILURE); }, 120000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex baeb4edd9..360ab6729 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-10-01-audio-off-permissions",+ "dataset_version": "2026-10-01-markdown-remote-images","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -4073,6 +4073,61 @@"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-html.config.mjs components/chat/eval-html-outcome.test.tsx"}},+ {+ "id": "app-markdown-remote-images",+ "name": "Render only local images and media from untrusted Markdown",+ "source": {+ "kind": "git_regression",+ "fix_commit": "7ee2e6ef078200f9781a98c39347d7c8f48c7663"+ },+ "base_ref": "7274f29b6b1bb940332b4e912b9cfcb20d591d3d",+ "oracle_ref": "7ee2e6ef078200f9781a98c39347d7c8f48c7663",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/markdown.tsx",+ "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx",+ "apps/screenpipe-app-tauri/app/notification-panel/page.tsx",+ "apps/screenpipe-app-tauri/components/notification-bell.tsx"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/markdown.tsx",+ "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "chat",+ "markdown",+ "privacy",+ "preserved-behavior"+ ],+ "prompt": "Rendered Markdown makes the application load remote resources the moment a message renders. Chat replies and summaries are built from captured outside content, so a steered reply can put private data in an image URL that leaks without any click. Repair the chat MarkdownBlock for both user and assistant messages and the shared MemoizedReactMarkdown renderer. Only local files may display as images or media: absolute paths, file URLs and Windows drive paths for images, plus home-relative paths for media, must still be read through the supplied native media port and displayed. No other image source (http, https, protocol-relative, relative, reference-style, raw HTML img, or picture/source srcset) may produce an element that loads it; keep its alt text visible instead. Media images, links and inline code that name a remote URL or a network-share path must never reach the native media reader; keep their alt text, link or code visible. Keep the surrounding message text. Keep the public component and native port contracts compatible. No particular sanitizer, helper or internal implementation is required. This task checks actual React/Markdown rendering with synthetic native ports; it does not establish webview CSP, mermaid or rich-text editor behavior, native file access policy or network isolation.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ },+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "packages/workflows-ui/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 90,+ "fixtures": [+ {+ "local_path": "graders/markdown-remote-image-outcomes.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/components/chat/eval-remote-image-outcome.test.tsx"+ },+ {+ "local_path": "graders/markdown-remote-image-outcomes.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-remote-images.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-remote-images.config.mjs components/chat/eval-remote-image-outcome.test.tsx"+ }+ },{"id": "ai-gateway-trusted-runner-admission","name": "Admit verified machine chat while preserving access refusals",diff --git a/evals/coding-agent/graders/markdown-remote-image-outcomes.config.mjs b/evals/coding-agent/graders/markdown-remote-image-outcomes.config.mjsnew file mode 100644index 000000000..2cbbd6680--- /dev/null+++ b/evals/coding-agent/graders/markdown-remote-image-outcomes.config.mjs@@ -0,0 +1,8 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {mergeConfig} from 'vitest/config';+import {fileURLToPath} from 'node:url';+import base from './vitest.config';+// Resolve the historical workspace package from the evaluated tree, not a+// cached node_modules workspace symlink. External packages remain matched.+export default mergeConfig(base, {resolve:{alias:{'@screenpipe/workflows-ui/chat':fileURLToPath(new URL('../../packages/workflows-ui/src/chat-primitives.tsx',import.meta.url))}},test:{include:['components/chat/eval-remote-image-outcome.test.tsx'],maxWorkers:1,minWorkers:1}});diff --git a/evals/coding-agent/graders/markdown-remote-image-outcomes.fixture.tsx.txt b/evals/coding-agent/graders/markdown-remote-image-outcomes.fixture.tsx.txtnew file mode 100644index 000000000..2d5205092--- /dev/null+++ b/evals/coding-agent/graders/markdown-remote-image-outcomes.fixture.tsx.txt@@ -0,0 +1,144 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { act, render, screen, waitFor } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { MarkdownBlock } from "./markdown-block";+import { MemoizedReactMarkdown } from "@/components/markdown";++const GIF = "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7";++const { readMediaMock, ownedBrowserNavigateMock, copyTextToClipboardMock, openViewerWindowMock, showWindowMock } =+ vi.hoisted(() => ({+ // Synthetic native media reader: the only way a local file reaches the page.+ readMediaMock: vi.fn(async (_path: string) => ({+ status: "ok" as const,+ data: { data: "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", mimeType: "image/gif" },+ })),+ ownedBrowserNavigateMock: vi.fn(async () => ({ status: "ok" as const, data: null })),+ copyTextToClipboardMock: vi.fn(async () => ({ status: "ok" as const, data: null })),+ openViewerWindowMock: vi.fn(async () => ({ status: "ok" as const })),+ showWindowMock: vi.fn(async () => ({ status: "ok" as const })),+ }));++vi.mock("@/lib/utils/tauri", () => ({+ commands: {+ getMediaFile: readMediaMock,+ ownedBrowserNavigate: ownedBrowserNavigateMock,+ copyTextToClipboard: copyTextToClipboardMock,+ openViewerWindow: openViewerWindowMock,+ showWindow: showWindowMock,+ },+}));+vi.mock("@tauri-apps/plugin-opener", () => ({ openUrl: vi.fn(async () => undefined) }));+vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));+vi.mock("@tauri-apps/api/event", () => ({ emit: vi.fn(async () => undefined) }));+vi.mock("@/lib/hooks/use-timeline-store", () => ({+ useTimelineStore: { getState: () => ({ setPendingNavigation: vi.fn() }) },+}));+vi.mock("@/lib/notifications/actions", () => ({ routeNotificationDeeplink: vi.fn(async () => undefined) }));++vi.hoisted(() => { globalThis.fetch = (() => { throw new Error("Eval network denied"); }) as typeof fetch; });++type Renderer = (text: string) => HTMLElement;+const chatAssistant: Renderer = (text) => render(<MarkdownBlock text={text} isUser={false} />).container;+const chatUser: Renderer = (text) => render(<MarkdownBlock text={text} isUser />).container;+const shared: Renderer = (text) => render(<MemoizedReactMarkdown>{text}</MemoizedReactMarkdown>).container;+const CHAT: [string, Renderer][] = [["chat assistant", chatAssistant], ["chat user", chatUser]];+const RENDERERS: [string, Renderer][] = [...CHAT, ["shared markdown", shared]];++// Elements that make the webview request something as soon as they render.+// Local images arrive as blob: URLs after the native reader returns bytes.+function fetchingElements(container: HTMLElement): string[] {+ return [...container.querySelectorAll("img, picture, source, video, audio, [srcset]")]+ .filter((element) => !(element.tagName === "IMG" && element.getAttribute("src")?.startsWith("blob:")))+ .map((element) => element.outerHTML);+}++async function settle() {+ await act(async () => { await new Promise((resolve) => setTimeout(resolve, 0)); });+}++beforeEach(() => {+ let next = 0;+ URL.createObjectURL = vi.fn(() => `blob:eval-${++next}`);+ URL.revokeObjectURL = vi.fn();+});+afterEach(() => { vi.clearAllMocks(); });++const BLOCKED_MARKDOWN: [string, string, string][] = [+ ["https image", "", "remote chart"],+ ["http image", "", "remote chart"],+ ["protocol-relative image", "", "remote chart"],+ ["reference-style image", "![remote chart][r]\n\n[r]: https://example.com/x.png", "remote chart"],+ ["relative image", "", "remote chart"],+ ["network-share image", "", "share chart"],+ ["remote media image", "", "clip"],+ ["remote media link", "[clip](https://example.com/a.mp4)", "clip"],+ ["remote media code", "`https://example.com/a.mp4`", "https://example.com/a.mp4"],+ ["network-share media image", "", "clip"],+ ["network-share media code", "`\\\\host\\share\\a.mp4`", "\\\\host\\share\\a.mp4"],+];++describe("remote markdown sources", () => {+ describe.each(RENDERERS)("%s", (_renderer, renderMarkdown) => {+ it.each(BLOCKED_MARKDOWN)("does not load a %s and keeps its text", async (_case, markdown, text) => {+ const container = renderMarkdown(`before\n\n${markdown}\n\nafter`);+ await settle();+ expect(container).toHaveTextContent("before");+ expect(container).toHaveTextContent(text);+ expect(container).toHaveTextContent("after");+ expect(fetchingElements(container)).toEqual([]);+ expect(readMediaMock).not.toHaveBeenCalled();+ });+ });++ describe.each(CHAT)("%s raw HTML", (renderer, renderChat) => {+ it.each([+ ["remote img", `<img src="https://example.com/x.png?d=secret" alt="remote chart">`],+ ["protocol-relative img", `<img src="//example.com/x.png" alt="remote chart">`],+ ])("does not load a %s and keeps its alt text", async (_case, html) => {+ const container = renderChat(`before\n\n${html}\n\nafter`);+ await settle();+ expect(container).toHaveTextContent("before");+ expect(container).toHaveTextContent("remote chart");+ expect(container).toHaveTextContent("after");+ expect(fetchingElements(container)).toEqual([]);+ expect(readMediaMock).not.toHaveBeenCalled();+ });++ it("keeps a picture source from replacing the local image beside it", async () => {+ const path = `/tmp/eval-picture-${renderer.replace(/\W/g, "-")}.gif`;+ const container = renderChat(+ `before\n\n<picture><source srcset="https://example.com/x.png?d=secret"><img src="${path}" alt="pic"></picture>\n\nafter`,+ );+ expect(await screen.findByAltText("pic")).toHaveAttribute("src", expect.stringMatching(/^blob:/));+ expect(readMediaMock).toHaveBeenCalledWith(path);+ expect(container).toHaveTextContent("before");+ expect(container).toHaveTextContent("after");+ expect(fetchingElements(container)).toEqual([]);+ });+ });+});++describe("local markdown files", () => {+ it.each([+ ["absolute image in chat", chatAssistant, "", "/tmp/eval-absolute.gif"],+ ["file URL image in chat", chatAssistant, "", "/tmp/eval-file-url.gif"],+ ["Windows image in chat", chatAssistant, "", "C:\\Users\\me\\eval-windows.gif"],+ ["absolute image in shared markdown", shared, "", "/tmp/eval-shared.gif"],+ ] as [string, Renderer, string, string][])("still shows an %s through the native reader", async (_case, renderMarkdown, markdown, path) => {+ renderMarkdown(markdown);+ expect(await screen.findByAltText("local chart")).toHaveAttribute("src", expect.stringMatching(/^blob:/));+ expect(readMediaMock).toHaveBeenCalledWith(path);+ });++ it.each([+ ["absolute media image in chat", chatAssistant, "", "/Users/me/eval-absolute.mp4"],+ ["home-relative media code in chat", chatAssistant, "`~/Downloads/eval-home.mp4`", "~/Downloads/eval-home.mp4"],+ ["absolute media link in shared markdown", shared, "[clip](/Users/me/eval-shared.mp4)", "/Users/me/eval-shared.mp4"],+ ] as [string, Renderer, string, string][])("still reads %s through the native reader", async (_case, renderMarkdown, markdown, path) => {+ renderMarkdown(markdown);+ await waitFor(() => expect(readMediaMock).toHaveBeenCalledWith(path));+ });+});