The Open Framework for Building Agents
fix(docs): include custom components in Markdown exports (#4330)
Colton Padden45bf86007348
diff --git a/apps/docs/lib/evals/eve-code-format.test.ts b/apps/docs/lib/evals/eve-code-format.test.tsnew file mode 100644index 000000000..0f3ba5baa--- /dev/null+++ b/apps/docs/lib/evals/eve-code-format.test.ts@@ -0,0 +1,55 @@+import { describe, expect, it } from "vitest";+import { renderBenchmarkMarkdown } from "./eve-code-format";+import type { EveCodeDatasetResults, EveCodeHarnessScore } from "./eve-code-results";++const score: EveCodeHarnessScore = {+ harness: "alice",+ version: null,+ resolved: 2,+ attempts: 3,+ resolveRate: { estimate: 2 / 3, low: 0.5, high: 0.8 },+ costUsd: null,+ costPerResolvedUsd: null,+ inputTokens: 0,+ outputTokens: 0,+ cachedShare: null,+ latencyP50Ms: 1250,+ latencyP90Ms: 2400,+ source: null,+ runUrl: null,+ measuredAt: "2026-10-03T12:00:00Z",+};++const results: EveCodeDatasetResults = {+ dataset: { name: "example", version: "v1" },+ model: { id: "example-model" },+ attempts: 3,+ tasks: 1,+ generatedAt: "2026-10-04T12:00:00Z",+ harnesses: [+ { ...score, harness: "bob", latencyP50Ms: 2000, measuredAt: null },+ {+ ...score,+ harness: "charlie",+ resolveRate: { estimate: 0.5, low: 0.2, high: 0.7 },+ latencyP50Ms: 500,+ },+ score,+ ],+};++describe("benchmark Markdown table", () => {+ it("formats the snapshot and orders by resolve rate, then latency", () => {+ expect(renderBenchmarkMarkdown(results, "example")).toBe(+ [+ "1 tasks × 3 attempts on `example-model`. Snapshot updated 2026-10-04.",+ "",+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",+ "| --- | --- | --- | --- | --- | --- |",+ "| `alice` | 67% (2/3) | 50%–80% | 1.3s | 2.4s | 2026-10-03 |",+ "| `bob` | 67% (2/3) | 50%–80% | 2.0s | 2.4s | — |",+ "| `charlie` | 50% (2/3) | 20%–70% | 0.5s | 2.4s | 2026-10-03 |",+ ].join("\n"),+ );+ });+});diff --git a/apps/docs/lib/evals/eve-code-format.ts b/apps/docs/lib/evals/eve-code-format.tsnew file mode 100644index 000000000..7a59d31f9--- /dev/null+++ b/apps/docs/lib/evals/eve-code-format.ts@@ -0,0 +1,29 @@+import type { EveCodeDatasetResults, EveCodeHarnessScore } from "./eve-code-results";++/** Ranks by resolve rate, then by median latency, matching eve-bench's correctness-first order. */+export function rankedHarnesses(results: EveCodeDatasetResults): EveCodeHarnessScore[] {+ return [...results.harnesses].sort(+ (left, right) =>+ right.resolveRate.estimate - left.resolveRate.estimate ||+ left.latencyP50Ms - right.latencyP50Ms,+ );+}++export function renderBenchmarkMarkdown(+ results: EveCodeDatasetResults | undefined,+ dataset: string,+): string {+ if (!results) return `No published ${dataset} results yet.`;+ const pct = (value: number) => `${Math.round(value * 100)}%`;+ const rows = rankedHarnesses(results).map(+ (h) =>+ `| \`${h.harness}\` | ${pct(h.resolveRate.estimate)} (${h.resolved}/${h.attempts}) | ${pct(h.resolveRate.low)}–${pct(h.resolveRate.high)} | ${(h.latencyP50Ms / 1000).toFixed(1)}s | ${(h.latencyP90Ms / 1000).toFixed(1)}s | ${h.measuredAt?.slice(0, 10) ?? "—"} |`,+ );+ return [+ `${results.tasks} tasks × ${results.attempts} attempts on \`${results.model.id}\`. Snapshot updated ${results.generatedAt.slice(0, 10)}.`,+ "",+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",+ "| --- | --- | --- | --- | --- | --- |",+ ...rows,+ ].join("\n");+}diff --git a/apps/docs/lib/evals/eve-code-results.ts b/apps/docs/lib/evals/eve-code-results.tsindex 977d68a38..5ed565fb6 100644--- a/apps/docs/lib/evals/eve-code-results.ts+++ b/apps/docs/lib/evals/eve-code-results.ts@@ -36,11 +36,4 @@ export interface EveCodeBenchmarkSnapshot {export const eveCodeBenchmark = rawSnapshot as EveCodeBenchmarkSnapshot;-/** Ranks by resolve rate, then by median latency, matching eve-bench's correctness-first order. */-export function rankedHarnesses(results: EveCodeDatasetResults): EveCodeHarnessScore[] {- return [...results.harnesses].sort(- (left, right) =>- right.resolveRate.estimate - left.resolveRate.estimate ||- left.latencyP50Ms - right.latencyP50Ms,- );-}+export { rankedHarnesses } from "./eve-code-format";diff --git a/apps/docs/lib/geistdocs/remark-benchmark-markdown.test.ts b/apps/docs/lib/geistdocs/remark-benchmark-markdown.test.tsnew file mode 100644index 000000000..826ec8381--- /dev/null+++ b/apps/docs/lib/geistdocs/remark-benchmark-markdown.test.ts@@ -0,0 +1,60 @@+import { createProcessor } from "@mdx-js/mdx";+import { toMarkdown, type Options } from "mdast-util-to-markdown";+import { describe, expect, it } from "vitest";+import config from "../../source.config";+import { remarkBenchmarkMarkdown } from "./remark-benchmark-markdown";++describe("benchmark Markdown export", () => {+ it.each(["<EveCodeBenchmark />", '<EveCodeBenchmark dataset={"example"} />'])(+ "rejects unsupported dataset attributes: %s",+ (source) => {+ const processor = createProcessor({ remarkPlugins: [remarkBenchmarkMarkdown] });+ const tree = processor.parse(source);+ expect(() =>+ toMarkdown(tree, {+ extensions: (processor.data() as { toMarkdownExtensions?: Options[] })+ .toMarkdownExtensions,+ }),+ ).toThrow("EveCodeBenchmark Markdown export requires a literal dataset string.");+ },+ );+ it("exports results without changing HTML components or fenced examples", async () => {+ const options =+ typeof config.mdxOptions === "function" ? await config.mdxOptions() : config.mdxOptions;+ const plugins = options?.remarkPlugins;+ if (!Array.isArray(plugins)) throw new Error("Expected configured remark plugins");+ const processor = createProcessor({ remarkPlugins: plugins });+ const tree = processor.parse(+ [+ '<EveCodeBenchmark dataset="deepswe-lean" />',+ "",+ "<EveCodeBenchmark",+ ' className="wide"',+ " dataset='not-published'",+ "/>",+ "",+ "<Callout>Keep this content.</Callout>",+ "",+ "```mdx",+ '<EveCodeBenchmark dataset="deepswe-lean" />',+ "```",+ ].join("\n"),+ );+ const markdown = toMarkdown(tree, {+ extensions: (processor.data() as { toMarkdownExtensions?: Options[] }).toMarkdownExtensions,+ });++ expect(markdown).toContain(+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",+ );+ expect(markdown).toContain("No published not-published results yet.");+ expect(markdown).toContain("<Callout>Keep this content.</Callout>");+ expect(markdown).toContain('```mdx\n<EveCodeBenchmark dataset="deepswe-lean" />\n```');+ expect(markdown.match(/<EveCodeBenchmark/g)).toHaveLength(1);+ expect(+ tree.children.filter(+ (node) => node.type === "mdxJsxFlowElement" && node.name === "EveCodeBenchmark",+ ),+ ).toHaveLength(2);+ });+});diff --git a/apps/docs/lib/geistdocs/remark-benchmark-markdown.ts b/apps/docs/lib/geistdocs/remark-benchmark-markdown.tsnew file mode 100644index 000000000..bedf1f3a4--- /dev/null+++ b/apps/docs/lib/geistdocs/remark-benchmark-markdown.ts@@ -0,0 +1,25 @@+import { mdxJsxToMarkdown, type MdxJsxFlowElement } from "mdast-util-mdx-jsx";+import type { Options } from "mdast-util-to-markdown";+import { renderBenchmarkMarkdown } from "../evals/eve-code-format";+import { eveCodeBenchmark } from "../evals/eve-code-results";++// Customize serialization only: HTML must retain the interactive component.+export function remarkBenchmarkMarkdown(this: { data(): unknown }): void {+ const data = this.data() as { toMarkdownExtensions?: Options[] };+ const extensions = (data.toMarkdownExtensions ??= []);+ const fallback = mdxJsxToMarkdown().handlers!.mdxJsxFlowElement!;+ extensions.push({+ handlers: {+ mdxJsxFlowElement(node: MdxJsxFlowElement, parent, state, info) {+ if (node.name !== "EveCodeBenchmark") return fallback(node, parent, state, info);+ const dataset = node.attributes.find(+ (attribute) => attribute.type === "mdxJsxAttribute" && attribute.name === "dataset",+ );+ if (dataset?.type !== "mdxJsxAttribute" || typeof dataset.value !== "string") {+ throw new Error("EveCodeBenchmark Markdown export requires a literal dataset string.");+ }+ return renderBenchmarkMarkdown(eveCodeBenchmark.datasets[dataset.value], dataset.value);+ },+ },+ });+}diff --git a/apps/docs/lib/geistdocs/remark-component-markdown.test.ts b/apps/docs/lib/geistdocs/remark-component-markdown.test.tsindex 5a1ae091e..b815e62f7 100644--- a/apps/docs/lib/geistdocs/remark-component-markdown.test.ts+++ b/apps/docs/lib/geistdocs/remark-component-markdown.test.ts@@ -1,6 +1,7 @@import { createProcessor } from "@mdx-js/mdx";import { toMarkdown, type Options } from "mdast-util-to-markdown";import { describe, expect, it } from "vitest";+import { remarkComponentMarkdown } from "./remark-component-markdown";import config from "../../source.config";describe("component Markdown export", () => {@@ -17,6 +18,8 @@ describe("component Markdown export", () => {"","<Callout>Keep this content.</Callout>","",+ '<EveCodeBenchmark dataset="deepswe-lean" />',+ "","```mdx","<AgentRuntimeDiagram />","```",@@ -51,7 +54,12 @@ describe("component Markdown export", () => {expect(markdown).toContain("**Workspace** — Persistent per-session files");expect(markdown).toContain("`/workspace`");expect(markdown).toContain("`from agent/sandbox/workspace/**`");- expect(markdown).toContain("<Callout>Keep this content.</Callout>");+ expect(markdown).toContain(+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",+ );+ expect(markdown).toContain("<Callout>");+ expect(markdown).toContain("Keep this content.");+ expect(markdown).toContain("</Callout>");expect(markdown).toContain("```mdx\n<AgentRuntimeDiagram />\n```");expect(markdown.match(/<AgentRuntimeDiagram \/>/g)).toHaveLength(1);diff --git a/apps/docs/lib/geistdocs/remark-component-markdown.ts b/apps/docs/lib/geistdocs/remark-component-markdown.tsindex 2abfd1694..0008b2b22 100644--- a/apps/docs/lib/geistdocs/remark-component-markdown.ts+++ b/apps/docs/lib/geistdocs/remark-component-markdown.ts@@ -1,5 +1,7 @@-import { mdxJsxToMarkdown } from "mdast-util-mdx-jsx";+import { mdxJsxToMarkdown, type MdxJsxFlowElement } from "mdast-util-mdx-jsx";import type { Options } from "mdast-util-to-markdown";+import { renderBenchmarkMarkdown } from "../evals/eve-code-format";+import { eveCodeBenchmark } from "../evals/eve-code-results";import { renderAgentRuntimeDiagramMarkdown } from "./agent-runtime-diagram";// Customize serialization only: the MDX tree must keep the visual component for HTML.@@ -9,8 +11,17 @@ export function remarkComponentMarkdown(this: { data(): unknown }): void {const fallback = mdxJsxToMarkdown().handlers!.mdxJsxFlowElement!;extensions.push({handlers: {- mdxJsxFlowElement(node, parent, state, info) {+ mdxJsxFlowElement(node: MdxJsxFlowElement, parent, state, info) {if (node.name === "AgentRuntimeDiagram") return renderAgentRuntimeDiagramMarkdown();+ if (node.name === "EveCodeBenchmark") {+ const dataset = node.attributes.find(+ (attribute) => attribute.type === "mdxJsxAttribute" && attribute.name === "dataset",+ );+ if (dataset?.type !== "mdxJsxAttribute" || typeof dataset.value !== "string") {+ throw new Error("EveCodeBenchmark Markdown export requires a literal dataset string.");+ }+ return renderBenchmarkMarkdown(eveCodeBenchmark.datasets[dataset.value], dataset.value);+ }return fallback(node, parent, state, info);},},diff --git a/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.ts b/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.tsindex 3da209686..683da16b8 100644--- a/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.ts+++ b/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.ts@@ -1,21 +1,21 @@import { defineEval } from "eve/evals";-const INITIAL_TARGET = "005INITIALTARGET";-const CORRECTED_TARGET = "005CORRECTEDTARGET";+const INITIAL_TARGET = "Alice";+const CORRECTED_TARGET = "Bob";export default defineEval({tags: ["real-model"],- description: "A newer user correction supersedes the model-visible pending-approval projection.",+ description:+ "An updated plan uses the latest requested target while an earlier action awaits approval.",async test(t) {const turn = await t.send([- "I am the original requester and confirm the intended outcome:",- `set the target to explicit user id ${CORRECTED_TARGET}, not ${INITIAL_TARGET}.`,- "The correction is authorized. Please revise and submit the plan now.",+ `The change plan should now target ${CORRECTED_TARGET} instead of ${INITIAL_TARGET}.`,+ "Please update the plan and send the revised version.",].join("\n"),{clientContext: [- `Original request: present a change plan targeting user id ${INITIAL_TARGET}.`,+ `Alice asked for a change plan targeting ${INITIAL_TARGET}.`,["[Pending approvals]","The following tool calls are awaiting approval and have not executed:",diff --git a/packages/eve/scripts/copy-docs.mjs b/packages/eve/scripts/copy-docs.mjsindex 51873bb69..224db8203 100644--- a/packages/eve/scripts/copy-docs.mjs+++ b/packages/eve/scripts/copy-docs.mjs@@ -23,33 +23,15 @@ const page = await readFile(benchmarkPage, "utf8").catch((error) => {throw error;});if (page !== null) {+ const { renderBenchmarkMarkdown } =+ await import("../../../apps/docs/lib/evals/eve-code-format.ts");const snapshot = JSON.parse(await readFile(join(monorepoRoot, "apps/docs/lib/evals/eve-code-benchmark.json"), "utf8"),);await writeFile(benchmarkPage,page.replace(/<EveCodeBenchmark dataset="([^"]+)" \/>/gu, (_, dataset) =>- benchmarkTable(snapshot.datasets[dataset], dataset),+ renderBenchmarkMarkdown(snapshot.datasets[dataset], dataset),),);}--function benchmarkTable(results, dataset) {- if (!results) return `No published ${dataset} results yet.`;- const pct = (value) => `${Math.round(value * 100)}%`;- const rows = [...results.harnesses]- .sort(- (a, b) => b.resolveRate.estimate - a.resolveRate.estimate || a.latencyP50Ms - b.latencyP50Ms,- )- .map(- (h) =>- `| \`${h.harness}\` | ${pct(h.resolveRate.estimate)} (${h.resolved}/${h.attempts}) | ${pct(h.resolveRate.low)}–${pct(h.resolveRate.high)} | ${(h.latencyP50Ms / 1000).toFixed(1)}s | ${(h.latencyP90Ms / 1000).toFixed(1)}s | ${h.measuredAt?.slice(0, 10) ?? "—"} |`,- );- return [- `${results.tasks} tasks × ${results.attempts} attempts on \`${results.model.id}\`. Snapshot updated ${results.generatedAt.slice(0, 10)}.`,- "",- "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",- "| --- | --- | --- | --- | --- | --- |",- ...rows,- ].join("\n");-}