YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover unmarked HTML file previews
Louis Beaumont63608cc353b6
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 866851387..6d9a728fd 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1,5 +1,15 @@# Coding-agent regression evals+The unmarked HTML file-preview case executes ViewerFileContent with the actual+Markdown, code-renderer, iframe and sandbox helper sources. It checks full and+snippet previews, parsed content/CSP metadata, source toggling, path transitions,+Unicode names/content and preserved text/Markdown. Fourteen calibration controls+include unused-correct code, empty/comment-only output, unsafe origin, fake CSP,+lost toggling/Markdown, equivalent implementations and missing-source setup errors.+Dependencies and hidden fixtures are installed only when grading begins. Synthetic+native/media/translation ports and JSDOM do not prove browser network/IPC enforcement,+Brain-view integration, execution isolation or model improvement.+This is an agent eval suite, not a unit-test suite. Every case contains:- a sanitized task derived from an escaped product failure;diff --git a/evals/coding-agent/calibrate-html-viewer.test.js b/evals/coding-agent/calibrate-html-viewer.test.jsnew file mode 100644index 000000000..5b2807107--- /dev/null+++ b/evals/coding-agent/calibrate-html-viewer.test.js@@ -0,0 +1,44 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {dirname,join,resolve} from 'node:path';+import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,rmSync,symlinkSync} from 'node:fs';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..'),app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(x=>x.id==='app-unmarked-html-file-preview');+const root=mkdtempSync(join(tmpdir(),'html-viewer-calibration-')),archives=new Map();+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function replace(cwd,file,from,to){const p=join(cwd,app,file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ if(!archives.has(ref)){+ const paths=execFileSync('git',['ls-tree','--name-only',`${ref}:${app}`],{cwd:repo,encoding:'utf8'}).trim().split('\n').filter(x=>!['src-tauri','public','e2e','.e2e'].includes(x)&&!x.startsWith('.env')).map(x=>`${app}/${x}`);+ archives.set(ref,execFileSync('git',['archive',ref,...paths],{cwd:repo,maxBuffer:128*1024*1024}));+ }+ execFileSync('tar',['-x','-C',cwd],{input:archives.get(ref)});+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ for(const link of item.grader_dependency_links){const dest=join(cwd,link.destination_path);mkdirSync(dirname(dest),{recursive:true});symlinkSync(join(repo,link.source_path),dest,'dir');}+ mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit:r.status,signal:r.signal,error:r.error?.message},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;+}+function passes(r){expect(r.status).toBe(0);expect(r.stderr).toContain('12 pass');expect(r.stderr).toContain('0 fail');}+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received).toBe(expected)');expect(r.stdout+r.stderr).not.toMatch(/Unhandled error|ModuleNotFound|Cannot find module/);}+const viewer='components/file-viewer.tsx',frame='components/file-viewer-html-frame.tsx';+test('parent has five intended failures and seven preserved passes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('7 pass');},60000);+test('historical reference passes',()=>passes(grade('reference')),60000);+test('current source passes',()=>passes(grade('current','HEAD')),60000);+test('unused correct component does not hide broken caller',()=>fails(grade('unused',item.oracle_ref,cwd=>{writeFileSync(join(cwd,app,'unused-correct-viewer.tsx'),readFileSync(join(cwd,app,viewer)));writeFileSync(join(cwd,app,viewer),execFileSync('git',['show',`${item.base_ref}:${app}/${viewer}`],{cwd:repo}));})),60000);+test('empty iframe loses meaningful content',()=>fails(grade('empty',item.oracle_ref,cwd=>replace(cwd,viewer,'html={content.text}','html={""}'))),60000);+test('same-origin sandbox is rejected',()=>fails(grade('unsafe-origin',item.oracle_ref,cwd=>replace(cwd,frame,' sandbox="allow-scripts"',' sandbox="allow-scripts allow-same-origin"'))),60000);+test('source toggle must work',()=>fails(grade('no-source-toggle',item.oracle_ref,cwd=>replace(cwd,viewer,'onClick={() => setShowRendered((v) => !v)}','onClick={() => {}}'))),60000);+test('missing CSP is rejected',()=>fails(grade('no-csp',item.oracle_ref,cwd=>replace(cwd,frame,'wrapHtmlForSandbox(html, { theme })','html'))),60000);+test('sandbox token whitespace is equivalent',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,frame,' sandbox="allow-scripts"',' sandbox=" allow-scripts "'))),60000);+test('commented payload is not content',()=>fails(grade('comment-payload',item.oracle_ref,cwd=>replace(cwd,frame,'srcDoc={srcDoc}','srcDoc={"<!-- " + srcDoc.replaceAll("-->", "") + " -->"}'))),60000);+test('CSP comment is not a policy',()=>fails(grade('fake-csp',item.oracle_ref,cwd=>replace(cwd,frame,'srcDoc={srcDoc}',`srcDoc={"<!-- default-src 'none'; connect-src 'none'; form-action 'none' -->" + html}`))),60000);+test('ordinary Markdown must survive',()=>fails(grade('preserved-markdown',item.oracle_ref,cwd=>replace(cwd,viewer,' {renderedText}\n </MemoizedReactMarkdown>',' {""}\n </MemoizedReactMarkdown>'))),60000);+test('equivalent default-selection implementation passes',()=>passes(grade('equivalent-inline',item.oracle_ref,cwd=>replace(cwd,'lib/utils/html-sandbox.ts','return hasHumanRenderMarker(text) || looksLikeFullHtmlDocument(text);','return /<!doctype\\s+html\\b|<html[\\s>]|<head[\\s>]|<body[\\s>]|screenpipe:render=human/i.test(text);'))),60000);+test('missing component is a setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,app,viewer)));expect(r.status).toBe(1);expect(r.stderr).toMatch(/ModuleNotFound|Unhandled error/);expect(r.stdout).not.toContain('EVAL_OBSERVATIONS');},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex bd327a27b..179a50906 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5156,6 +5156,50 @@],"command": "bun --no-env-file test auth-host.eval.test.tsx"}+ },+ {+ "id": "app-unmarked-html-file-preview",+ "name": "Render unmarked HTML files without losing source or ordinary previews",+ "source": {+ "kind": "git_regression",+ "fix_commit": "93c0de3c6b9b872056987ef70bf46c9af2d9d890"+ },+ "base_ref": "6ea84d80e5ab4041aa855014ab71542a9d353206",+ "oracle_ref": "93c0de3c6b9b872056987ef70bf46c9af2d9d890",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/file-viewer.tsx",+ "apps/screenpipe-app-tauri/lib/utils/html-sandbox.ts"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/file-viewer.tsx",+ "apps/screenpipe-app-tauri/components/file-viewer-html-frame.tsx",+ "apps/screenpipe-app-tauri/lib/utils/html-sandbox.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "html",+ "preserved-behavior"+ ],+ "prompt": "An HTML report without a producer marker is only shown as source in the file viewer. Repair the actual ViewerFileContent preview so complete unmarked HTML documents open rendered, including upper-case .HTM names and Unicode content. Nonempty bare snippets must offer rendering too; their initial tab is unspecified. Preserve rendering for marked snippets. Keep meaningful document content inside a sandboxed iframe, with no payload style/script/heading elements inserted into the host. Grant at most script execution in the sandbox, and retain actual default-deny CSP metadata for default sources, connections and form submissions. Commented-out content or policy text is not a rendered document or policy. The source/render switch must show the unchanged source and return to the preview. Keep a chosen source tab when the same path is rerendered; a new path must clear old content while loading and display its own document afterward. Truncated, empty and loading HTML must not render or offer a render toggle. Preserve literal plain-text previews and ordinary Markdown headings, bold text, lists and safe web links. Keep the public component and existing source/render control contracts compatible. No specific internal helper or default-selection implementation is required. This task checks actual React/Markdown DOM output and iframe configuration with synthetic native/media ports; it does not establish browser network or IPC enforcement, Brain-view behavior or native security.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/html-viewer.fixture.ts.txt",+ "destination_path": "evals/coding-agent/graders/html-viewer.test.ts"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/html-viewer.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/html-viewer.fixture.ts.txt b/evals/coding-agent/graders/html-viewer.fixture.ts.txtnew file mode 100644index 000000000..9b4080c7f--- /dev/null+++ b/evals/coding-agent/graders/html-viewer.fixture.ts.txt@@ -0,0 +1,90 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {test, expect} from "bun:test";+import { createRequire } from 'node:module';+import { resolve, dirname } from 'node:path';+import { mkdtempSync, rmSync } from 'node:fs';+import {tmpdir} from 'node:os';+const buildDir=mkdtempSync(resolve(tmpdir(),'html-viewer-grader-'));+const runtime=resolve('apps/screenpipe-app-tauri/node_modules');+const require=createRequire(runtime+'/package.json');+const React=require('react');const {JSDOM}=require('jsdom');+const dom=new JSDOM('<!doctype html><html><body></body></html>',{url:'http://localhost/'});+for(const key of ['window','document','navigator','HTMLElement','HTMLIFrameElement','Node','Event','MouseEvent','MutationObserver'])Object.defineProperty(globalThis,key,{value:dom.window[key],configurable:true});+globalThis.React=React;globalThis.IS_REACT_ACT_ENVIRONMENT=true;+const {createRoot}=require('react-dom/client');const {act}=React;+const stubs={+ '@tauri-apps/plugin-shell':'export const open=async()=>{};',+ '@/lib/utils/tauri':'export const commands={openNotePath:async()=>{},readViewerFile:async()=>{throw new Error("unexpected native read")}};',+ '@/lib/utils':'export const cn=(...x)=>x.filter(Boolean).join(" ");',+ '@/lib/hooks/use-platform':'export const usePlatform=()=>({isMac:false});',+ 'gt-react':'export const useGT=()=>((s)=>s);',+ '@/lib/hooks/use-event-listener':'import {useEffect} from "react";export const useEventListener=(name,cb)=>useEffect(()=>{window.addEventListener(name,cb);return()=>window.removeEventListener(name,cb)},[name,cb]);',+};+Object.assign(stubs,{+ '@/components/rewind/media':'export const MediaComponent=()=>null;',+ '@/components/markdown/local-markdown-image':'export const LocalMarkdownImage=()=>null;',+ '@/lib/api':'export const getApiBaseUrl=()=>"http://127.0.0.1:0";',+ '@tauri-apps/api/core':'export const convertFileSrc=x=>x;'+});+globalThis.fetch=()=>{throw new Error('Network denied')};+const results=[];+{+ const arm='candidate';+ const rootPath=resolve('apps/screenpipe-app-tauri');+ const build=await Bun.build({entrypoints:[rootPath+'/components/file-viewer.tsx'],outdir:buildDir,target:'node',format:'esm',plugins:[{name:'reviewed-test-ports',setup(b){+ b.onResolve({filter:/.*/},a=>{+ if(a.path==='react'||a.path.startsWith('react/'))return {path:require.resolve(a.path),external:true};+ if(a.path==='@/lib/utils/html-sandbox')return {path:rootPath+'/lib/utils/html-sandbox.ts'};+ if(stubs[a.path])return {path:a.path,namespace:'port'};+ if(a.path.startsWith('@/')){const path=rootPath+'/'+a.path.slice(2); for(const ext of ['.tsx','.ts','.js'])if(require('node:fs').existsSync(path+ext))return {path:path+ext};}+ if(!a.path.startsWith('.')&&!a.path.startsWith('/')){try{return {path:require.resolve(a.path),external:true};}catch{}}+ });+ b.onLoad({filter:/.*/,namespace:'port'},a=>({contents:stubs[a.path],loader:'js'}));+ }}]});+ if(!build.success)throw new Error(JSON.stringify(build.logs));+ const {ViewerFileContent}=await import(buildDir+'/file-viewer.js');+ const full='<!doctype html><html><head><style>body{background:rgb(1,2,3)}</style></head><body><h1>SYNTHETIC_REPORT_417</h1><script>window.syntheticPayload=417</script></body></html>';+ const tests=[['full',full,false,'report.html'],['full-unicode',full.replaceAll('SYNTHETIC_REPORT_417','SECOND_Ω_528'),false,'Other.HTM'],['marked','<!-- screenpipe:render=human --><h1>SYNTHETIC_REPORT_417</h1>',false,'report.html'],['snippet','<h1>SYNTHETIC_REPORT_417</h1>',false,'report.html'],['truncated',full,true,'report.html'],['empty','',false,'report.html'],['plaintext','literal <h1>SYNTHETIC_REPORT_417</h1>',false,'report.txt'],['loading',null,false,'report.html'],['markdown','# Report\n\n**Preserved bold** and [safe link](https://example.com)\n\n- first\n- second',false,'report.md']];+ for(const [name,text,truncated,filename] of tests){+ const container=document.createElement('div');document.body.append(container);const root=createRoot(container);+ const content=text===null?null:{kind:'text',text,name:filename,path:'/synthetic/'+filename,truncated,total_bytes:text.length};+ await act(async()=>root.render(React.createElement(ViewerFileContent,{path:'/synthetic/'+filename,content})));+ const iframe=()=>container.querySelector('iframe');const toggle=()=>container.querySelector('[data-testid="html-render-toggle"]');+ const row={arm,case:name,initial_frame:!!iframe(),toggle_present:!!toggle(),host_payload_nodes:container.querySelectorAll('style,script,h1').length};+ if(['full','full-unicode','marked','snippet'].includes(name)){+ if(!iframe()&&toggle())await act(async()=>toggle().click());+ const f=iframe();row.render_available=!!f;const parsed=new JSDOM(f?.getAttribute('srcdoc')??'');const doc=parsed.window.document;row.payload_preserved=doc.querySelector('h1')?.textContent===(name==='full-unicode'?'SECOND_Ω_528':'SYNTHETIC_REPORT_417');+ row.sandbox=f?.getAttribute('sandbox')??null;const policies=Array.from(doc.querySelectorAll('meta[http-equiv]')).filter(m=>m.getAttribute('http-equiv')?.toLowerCase()==='content-security-policy').map(m=>new Map((m.getAttribute('content')??'').split(';').map(d=>{const [key,...values]=d.trim().split(/\s+/);return[key,values]})));row.csp=['default-src','connect-src','form-action'].every(d=>policies.some(policy=>JSON.stringify(policy.get(d))===JSON.stringify(["'none'"])));parsed.window.close();+ if(f&&toggle()){+ await act(async()=>toggle().click());row.source_preserved=!iframe()&&container.textContent.includes(text);+ await act(async()=>toggle().click());row.return_rendered=!!iframe();+ }else{row.source_preserved=container.textContent.includes(text);row.return_rendered=false;}+ row.expected_pass=row.render_available&&row.payload_preserved&&row.sandbox!==null&&row.sandbox.split(/\s+/).filter(Boolean).every(x=>x==='allow-scripts')&&row.csp&&row.source_preserved&&row.return_rendered&&row.host_payload_nodes===0&&(!name.startsWith('full')||row.initial_frame);+ }else if(name==='markdown'){row.expected_pass=!!container.querySelector('h1')&&container.querySelector('strong')?.textContent==='Preserved bold'&&container.querySelectorAll('li').length===2&&container.querySelector('a')?.getAttribute('href')==='https://example.com'&&!iframe();}+ else row.expected_pass=!iframe()&&!toggle()&&row.host_payload_nodes===0&&(text===null||text===''||container.textContent.includes(text));+ results.push(row);await act(async()=>root.unmount());container.remove();+ }+}+{+ const rootPath=resolve('apps/screenpipe-app-tauri');+ const {ViewerFileContent}=await import(buildDir+'/file-viewer.js');+ const box=document.createElement('div');document.body.append(box);const root=createRoot(box);+ const pathA='/synthetic/first.html',pathB='/synthetic/second.html';+ const content=(path,label)=>({kind:'text',name:path.split('/').pop(),path,text:'<!doctype html><html><body><h1>'+label+'</h1></body></html>',truncated:false,total_bytes:99});+ const a=content(pathA,'FIRST_NAV_901'),b=content(pathB,'SECOND_NAV_902');+ const render=async(path,value)=>act(async()=>root.render(React.createElement(ViewerFileContent,{path,content:value})));+ const frame=()=>box.querySelector('iframe');const toggle=()=>box.querySelector('[data-testid="html-render-toggle"]');+ await render(pathA,null);await render(pathA,a);+ results.push({case:'loaded-document-default',expected_pass:!!frame()});+ if(frame()&&toggle())await act(async()=>toggle().click());+ await render(pathA,{...a});+ results.push({case:'same-path-keeps-source-choice',expected_pass:!frame()&&box.textContent.includes(a.text)});+ await render(pathB,null);const loadingCleared=!frame();await render(pathB,b);+ const parsed=new JSDOM(frame()?.getAttribute('srcdoc')??'');+ results.push({case:'new-path-loads-new-document',expected_pass:loadingCleared&&parsed.window.document.querySelector('h1')?.textContent==='SECOND_NAV_902'&&!box.textContent.includes('FIRST_NAV_901')});parsed.window.close();+ await act(async()=>root.unmount());box.remove();+}+rmSync(buildDir,{recursive:true,force:true});dom.window.close();+console.log('EVAL_OBSERVATIONS '+JSON.stringify(results));+for(const row of results)test(row.case,()=>expect(row.expected_pass).toBe(true));