A desktop coding agent with free access to frontier models.
TypeScript
import type { TestArtifact, TestRunRecord } from "./schema.ts";
import type { TestArtifactIndexEntry } from "./scan.ts";
export interface RenderPrOptions {
title?: string;
reproCommand?: string;
notice?: string;
sourcePath?: string;
}
function html(value: string): string {
return value
.replaceAll("&", "&")
.replaceAll("<", "<")
.replaceAll(">", ">")
.replaceAll('"', """)
.replaceAll("'", "'");
}
function artifactBadge(entry: TestArtifactIndexEntry): { label: string; className: string; summary: string } {
if (entry.kind === "legacy-artifacts") {
return {
label: "LEGACY",
className: "legacy",
summary: `${entry.pngFiles.length} loose screenshot${entry.pngFiles.length === 1 ? "" : "s"}`,
};
}
const summary = entry.testRun.summary;
if (entry.testRun.outcome === "failed") {
return { label: "FAILED", className: "failed", summary: entry.testRun.failure ?? "Test body failed" };
}
if (entry.testRun.outcome === "skipped") {
return { label: "SKIPPED", className: "pending", summary: entry.testRun.failure ?? "Test skipped" };
}
if (summary.failedArtifacts > 0 || summary.failedExpectations > 0) {
return {
label: "FAILED",
className: "failed",
summary: `${summary.passedArtifacts}/${summary.totalArtifacts} artifacts passed · ${summary.failedArtifacts} failed`,
};
}
if (summary.pendingArtifacts > 0 || summary.pendingJudgments > 0) {
return {
label: "PENDING",
className: "pending",
summary: `${summary.pendingArtifacts}/${summary.totalArtifacts} artifacts pending · ${summary.pendingJudgments} expectations pending`,
};
}
if (summary.ok) {
return {
label: "PASSED",
className: "passed",
summary: `${summary.passedArtifacts}/${summary.totalArtifacts} artifacts passed`,
};
}
if (entry.testRun.outcome === "passed") {
return { label: "PASSED", className: "passed", summary: "Provenance trace passed" };
}
return {
label: "UNVALIDATED",
className: "unvalidated",
summary: `${summary.unvalidatedArtifacts}/${summary.totalArtifacts} artifacts unvalidated`,
};
}
interface TraceGroup {
label: string;
entries: { detail: string; count: number }[];
}
function renderedTrace(testRun: TestRunRecord): TestRunRecord["trace"] {
return testRun.trace.filter((entry) => entry.verb !== "tmpPath");
}
function traceGroups(testRun: TestRunRecord): TraceGroup[] {
const groups: TraceGroup[] = [];
for (const entry of [...renderedTrace(testRun)].sort((left, right) => left.seq - right.seq)) {
if (entry.channel === "step") continue;
const label = entry.stage === "world" ? "world" : entry.channel;
let group = groups.at(-1);
if (group?.label !== label) {
group = { label, entries: [] };
groups.push(group);
}
const detail = `${entry.ok ? "" : "❌ "}${entry.detail}`;
const previous = group.entries.at(-1);
if (previous?.detail === detail) previous.count += 1;
else group.entries.push({ detail, count: 1 });
}
return groups;
}
function stepLine(testRun: TestRunRecord): string | null {
if (testRun.steps.length === 0) return null;
return testRun.steps.map((step, index) => {
const icon = step.ok === true ? "✅" : step.ok === false ? "❌" : "⏭";
const duration = step.ms === undefined ? "" : ` (${(step.ms / 1_000).toFixed(1)}s)`;
return `${index + 1} ${icon} ${step.name}${duration}`;
}).join(" · ");
}
function verdictLine(testRun: TestRunRecord): string {
const trace = renderedTrace(testRun);
const observations = trace.filter((entry) => (
(entry.channel === "user" && (entry.verb === "see" || entry.verb === "notSee" || entry.verb === "screenshot"))
|| entry.channel === "vision"
));
const observationCounts = new Map<string, number>();
for (const entry of observations) observationCounts.set(entry.verb, (observationCounts.get(entry.verb) ?? 0) + 1);
const observationDetail = [...observationCounts.entries()].map(([verb, count]) => `${verb} ×${count}`).join(", ");
const probes = trace.filter((entry) => entry.channel === "probe" || entry.channel === "probe:raw").length;
const passedSteps = testRun.steps.filter((step) => step.ok === true).length;
return `${testRun.outcome} · ${observations.length} user observations${observationDetail ? ` (${observationDetail})` : ""} · ${probes} probes · steps ${passedSteps}/${testRun.steps.length}`;
}
function traceMarkdown(testRun: TestRunRecord): string[] {
const lines = traceGroups(testRun).map((group) => (
`**[${group.label}]** ${group.entries.map((entry) => `${entry.detail}${entry.count > 1 ? ` ×${entry.count}` : ""}`).join(" · ")}`
));
const steps = stepLine(testRun);
if (steps) lines.push(`**steps** ${steps}`);
lines.push(`**verdict** ${verdictLine(testRun)}`);
return lines;
}
function traceHtml(testRun: TestRunRecord): string {
const lines = traceGroups(testRun).map((group) => (
`<div><strong>[${html(group.label)}]</strong> ${group.entries.map((entry) => `${html(entry.detail)}${entry.count > 1 ? ` ×${entry.count}` : ""}`).join(" · ")}</div>`
));
const steps = stepLine(testRun);
if (steps) lines.push(`<div><strong>steps</strong> ${html(steps)}</div>`);
lines.push(`<div><strong>verdict</strong> ${html(verdictLine(testRun))}</div>`);
return `<div class="trace">${lines.join("")}</div>`;
}
export function renderArtifactIndexHtml(entries: TestArtifactIndexEntry[]): string {
const ordered = [...entries].sort((left, right) => right.createdAt.localeCompare(left.createdAt));
const cards = ordered.map((entry) => {
const badge = artifactBadge(entry);
const caption = entry.kind === "test-run" ? entry.testRun.artifacts[0]?.caption : undefined;
const thumbnail = entry.thumbnailHref
? `<img src="${html(entry.thumbnailHref)}" alt="${html(caption ?? entry.name)}">`
: `<div class="empty">No screenshot recorded</div>`;
const provenance = entry.kind === "test-run" ? traceHtml(entry.testRun) : "";
return `<article class="card ${badge.className}">
<a class="thumb" href="${html(entry.href)}">${thumbnail}</a>
<div class="copy">
<div class="topline"><span class="badge">${badge.label}</span><time datetime="${html(entry.createdAt)}">${html(entry.createdAt)}</time></div>
<h2><a href="${html(entry.href)}">${html(entry.name)}</a></h2>
${caption ? `<p class="caption">${html(caption)}</p>` : ""}
<p class="summary">${html(badge.summary)}</p>
${provenance}
</div>
</article>`;
}).join("\n");
return `<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>OmniRush.ai test artifacts</title><style>
body{font:15px/1.5 system-ui,sans-serif;max-width:1100px;margin:40px auto;padding:0 20px;background:#f6f7f9;color:#17191d}header,.card{background:white;border:1px solid #dfe2e8;border-radius:12px;padding:20px;margin:0 0 24px}header h1{margin:0 0 6px}.muted,time,.summary{color:#636c76}.card{display:grid;grid-template-columns:minmax(220px,40%) 1fr;gap:20px}.card.passed{border-left:6px solid #238636}.card.failed{border-left:6px solid #cf222e}.card.pending,.card.unvalidated{border-left:6px solid #9a6700}.card.legacy{border-left:6px solid #656d76}.thumb img,.empty{display:block;width:100%;aspect-ratio:16/10;object-fit:cover;border:1px solid #dfe2e8;border-radius:8px;background:#f6f7f9}.empty{display:grid;place-items:center;color:#636c76}.topline{display:flex;align-items:center;gap:10px}.badge{font-size:12px;font-weight:700}.passed .badge{color:#1a7f37}.failed .badge{color:#cf222e}.pending .badge,.unvalidated .badge{color:#9a6700}.legacy .badge{color:#656d76}h2{margin:12px 0 6px}a{color:inherit}.caption{font-weight:600}.trace{font-size:13px}.trace div{margin:4px 0}.trace strong{display:inline-block;min-width:65px}@media(max-width:700px){.card{grid-template-columns:1fr}}
</style></head><body><header><h1>Test artifacts</h1><p class="muted">Test evidence captured as the user experienced it, newest first.</p></header>${cards || '<p class="muted">No test artifacts found.</p>'}</body></html>\n`;
}
function summaryLine(testRun: TestRunRecord): string {
const summary = testRun.summary;
const failed = testRun.outcome === "failed" || summary.failedArtifacts > 0 || summary.failedExpectations > 0;
const pending = summary.pendingArtifacts > 0 || summary.pendingJudgments > 0;
const icon = failed ? "❌" : testRun.outcome === "skipped" || pending ? "⏳" : testRun.outcome === "passed" || summary.ok ? "✅" : "⚪";
return `${icon} **${artifactVerdict(testRun)}** · ${summary.passedExpectations} expectations passed · ${summary.failedExpectations} failed · ${summary.pendingJudgments} pending`;
}
function outcomeHeading(testRun: TestRunRecord): string {
if (testRun.outcome === "passed") return "✅ passed";
if (testRun.outcome === "failed") return "❌ failed";
if (testRun.outcome === "skipped") return "⏭ skipped";
return artifactVerdict(testRun);
}
function renderArtifact(artifact: TestArtifact, sequence: number, imageUrl: string | undefined): string {
const assertion = artifact.fileName.length === 0;
const pending = artifact.judgments.some((judgment) => judgment.state === "pending");
const marker = pending ? "⏳ PENDING" : assertion ? (artifact.ok === false ? "❌ FAIL ASSERTION" : "ℹ️ ASSERTION") : artifact.ok === false ? "❌ FAIL" : artifact.ok === true ? "✅ PASS" : "⚪ UNVALIDATED";
const lines = [`### ${marker} — ${sequence}. ${html(artifact.caption)}`, ""];
if (assertion && artifact.description) lines.push(html(artifact.description), "");
if (artifact.judgments.length === 0) {
lines.push("- ⚪ **UNVALIDATED** — no visual expectations recorded.");
} else {
for (const judgment of artifact.judgments) {
const judgmentMarker = judgment.state === "passed" ? "✅ **PASS**" : judgment.state === "failed" ? "❌ **FAIL**" : "⏳ **PENDING**";
lines.push(`- ${judgmentMarker} ${html(judgment.expectation)} — ${html(judgment.reasoning)}`);
}
}
if (imageUrl) {
lines.push("", ``);
}
return lines.join("\n");
}
function artifactVerdict(testRun: TestRunRecord): string {
const screenshots = testRun.artifacts.filter((artifact) => artifact.fileName.length > 0);
if (screenshots.length === 0 && testRun.outcome !== "unknown") return testRun.outcome;
const passed = screenshots.filter((artifact) => artifact.ok === true).length;
const pending = screenshots.filter((artifact) => artifact.judgments.some((judgment) => judgment.state === "pending")).length;
const assertions = testRun.artifacts.length - screenshots.length;
return `${passed}/${screenshots.length} screenshots passed${pending > 0 ? ` · ${pending} pending` : ""}${assertions > 0 ? ` · ${assertions} assertion${assertions === 1 ? "" : "s"}` : ""}`;
}
export function renderPrMarkdown(
testRun: TestRunRecord,
urls: Record<string, string>,
opts: RenderPrOptions = {},
): string {
const title = opts.title ?? `Test evidence — ${testRun.name}`;
const lines = [
"<!-- test-evidence -->",
`## ${html(title)} — ${outcomeHeading(testRun)}`,
"",
`SHA ${html(testRun.gitSha ?? "unknown")} · engine ${testRun.engine}`,
"",
summaryLine(testRun),
];
if (opts.notice) lines.push("", `> ${opts.notice}`);
lines.push("", ...traceMarkdown(testRun));
for (const [index, artifact] of testRun.artifacts.entries()) {
lines.push("", renderArtifact(artifact, index + 1, urls[artifact.fileName]));
}
const source = opts.sourcePath ?? `evals/results/test-runs/${testRun.dir.split(/[\\/]/).at(-1) ?? testRun.name}/test-run.json`;
lines.push(
"",
"---",
`_Test run created ${html(testRun.createdAt)} · Source: \`${html(source)}\`${opts.reproCommand ? ` · Repro: \`${html(opts.reproCommand)}\`` : ""}_`,
"<!-- /test-evidence -->",
);
return lines.join("\n");
}