Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(docs): include custom components in Markdown exports (#4330)

Colton Padden45bf86007348
diff --git a/apps/docs/lib/evals/eve-code-format.test.ts b/apps/docs/lib/evals/eve-code-format.test.ts
new file mode 100644
index 000000000..0f3ba5baa
--- /dev/null
+++ b/apps/docs/lib/evals/eve-code-format.test.ts
@@ -0,0 +1,55 @@
+import { describe, expect, it } from "vitest";
+import { renderBenchmarkMarkdown } from "./eve-code-format";
+import type { EveCodeDatasetResults, EveCodeHarnessScore } from "./eve-code-results";
+
+const score: EveCodeHarnessScore = {
+ harness: "alice",
+ version: null,
+ resolved: 2,
+ attempts: 3,
+ resolveRate: { estimate: 2 / 3, low: 0.5, high: 0.8 },
+ costUsd: null,
+ costPerResolvedUsd: null,
+ inputTokens: 0,
+ outputTokens: 0,
+ cachedShare: null,
+ latencyP50Ms: 1250,
+ latencyP90Ms: 2400,
+ source: null,
+ runUrl: null,
+ measuredAt: "2026-10-03T12:00:00Z",
+};
+
+const results: EveCodeDatasetResults = {
+ dataset: { name: "example", version: "v1" },
+ model: { id: "example-model" },
+ attempts: 3,
+ tasks: 1,
+ generatedAt: "2026-10-04T12:00:00Z",
+ harnesses: [
+ { ...score, harness: "bob", latencyP50Ms: 2000, measuredAt: null },
+ {
+ ...score,
+ harness: "charlie",
+ resolveRate: { estimate: 0.5, low: 0.2, high: 0.7 },
+ latencyP50Ms: 500,
+ },
+ score,
+ ],
+};
+
+describe("benchmark Markdown table", () => {
+ it("formats the snapshot and orders by resolve rate, then latency", () => {
+ expect(renderBenchmarkMarkdown(results, "example")).toBe(
+ [
+ "1 tasks × 3 attempts on `example-model`. Snapshot updated 2026-10-04.",
+ "",
+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",
+ "| --- | --- | --- | --- | --- | --- |",
+ "| `alice` | 67% (2/3) | 50%–80% | 1.3s | 2.4s | 2026-10-03 |",
+ "| `bob` | 67% (2/3) | 50%–80% | 2.0s | 2.4s | — |",
+ "| `charlie` | 50% (2/3) | 20%–70% | 0.5s | 2.4s | 2026-10-03 |",
+ ].join("\n"),
+ );
+ });
+});
diff --git a/apps/docs/lib/evals/eve-code-format.ts b/apps/docs/lib/evals/eve-code-format.ts
new file mode 100644
index 000000000..7a59d31f9
--- /dev/null
+++ b/apps/docs/lib/evals/eve-code-format.ts
@@ -0,0 +1,29 @@
+import type { EveCodeDatasetResults, EveCodeHarnessScore } from "./eve-code-results";
+
+/** Ranks by resolve rate, then by median latency, matching eve-bench's correctness-first order. */
+export function rankedHarnesses(results: EveCodeDatasetResults): EveCodeHarnessScore[] {
+ return [...results.harnesses].sort(
+ (left, right) =>
+ right.resolveRate.estimate - left.resolveRate.estimate ||
+ left.latencyP50Ms - right.latencyP50Ms,
+ );
+}
+
+export function renderBenchmarkMarkdown(
+ results: EveCodeDatasetResults | undefined,
+ dataset: string,
+): string {
+ if (!results) return `No published ${dataset} results yet.`;
+ const pct = (value: number) => `${Math.round(value * 100)}%`;
+ const rows = rankedHarnesses(results).map(
+ (h) =>
+ `| \`${h.harness}\` | ${pct(h.resolveRate.estimate)} (${h.resolved}/${h.attempts}) | ${pct(h.resolveRate.low)}–${pct(h.resolveRate.high)} | ${(h.latencyP50Ms / 1000).toFixed(1)}s | ${(h.latencyP90Ms / 1000).toFixed(1)}s | ${h.measuredAt?.slice(0, 10) ?? "—"} |`,
+ );
+ return [
+ `${results.tasks} tasks × ${results.attempts} attempts on \`${results.model.id}\`. Snapshot updated ${results.generatedAt.slice(0, 10)}.`,
+ "",
+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",
+ "| --- | --- | --- | --- | --- | --- |",
+ ...rows,
+ ].join("\n");
+}
diff --git a/apps/docs/lib/evals/eve-code-results.ts b/apps/docs/lib/evals/eve-code-results.ts
index 977d68a38..5ed565fb6 100644
--- a/apps/docs/lib/evals/eve-code-results.ts
+++ b/apps/docs/lib/evals/eve-code-results.ts
@@ -36,11 +36,4 @@ export interface EveCodeBenchmarkSnapshot {
export const eveCodeBenchmark = rawSnapshot as EveCodeBenchmarkSnapshot;
-/** Ranks by resolve rate, then by median latency, matching eve-bench's correctness-first order. */
-export function rankedHarnesses(results: EveCodeDatasetResults): EveCodeHarnessScore[] {
- return [...results.harnesses].sort(
- (left, right) =>
- right.resolveRate.estimate - left.resolveRate.estimate ||
- left.latencyP50Ms - right.latencyP50Ms,
- );
-}
+export { rankedHarnesses } from "./eve-code-format";
diff --git a/apps/docs/lib/geistdocs/remark-benchmark-markdown.test.ts b/apps/docs/lib/geistdocs/remark-benchmark-markdown.test.ts
new file mode 100644
index 000000000..826ec8381
--- /dev/null
+++ b/apps/docs/lib/geistdocs/remark-benchmark-markdown.test.ts
@@ -0,0 +1,60 @@
+import { createProcessor } from "@mdx-js/mdx";
+import { toMarkdown, type Options } from "mdast-util-to-markdown";
+import { describe, expect, it } from "vitest";
+import config from "../../source.config";
+import { remarkBenchmarkMarkdown } from "./remark-benchmark-markdown";
+
+describe("benchmark Markdown export", () => {
+ it.each(["<EveCodeBenchmark />", '<EveCodeBenchmark dataset={"example"} />'])(
+ "rejects unsupported dataset attributes: %s",
+ (source) => {
+ const processor = createProcessor({ remarkPlugins: [remarkBenchmarkMarkdown] });
+ const tree = processor.parse(source);
+ expect(() =>
+ toMarkdown(tree, {
+ extensions: (processor.data() as { toMarkdownExtensions?: Options[] })
+ .toMarkdownExtensions,
+ }),
+ ).toThrow("EveCodeBenchmark Markdown export requires a literal dataset string.");
+ },
+ );
+ it("exports results without changing HTML components or fenced examples", async () => {
+ const options =
+ typeof config.mdxOptions === "function" ? await config.mdxOptions() : config.mdxOptions;
+ const plugins = options?.remarkPlugins;
+ if (!Array.isArray(plugins)) throw new Error("Expected configured remark plugins");
+ const processor = createProcessor({ remarkPlugins: plugins });
+ const tree = processor.parse(
+ [
+ '<EveCodeBenchmark dataset="deepswe-lean" />',
+ "",
+ "<EveCodeBenchmark",
+ ' className="wide"',
+ " dataset='not-published'",
+ "/>",
+ "",
+ "<Callout>Keep this content.</Callout>",
+ "",
+ "```mdx",
+ '<EveCodeBenchmark dataset="deepswe-lean" />',
+ "```",
+ ].join("\n"),
+ );
+ const markdown = toMarkdown(tree, {
+ extensions: (processor.data() as { toMarkdownExtensions?: Options[] }).toMarkdownExtensions,
+ });
+
+ expect(markdown).toContain(
+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",
+ );
+ expect(markdown).toContain("No published not-published results yet.");
+ expect(markdown).toContain("<Callout>Keep this content.</Callout>");
+ expect(markdown).toContain('```mdx\n<EveCodeBenchmark dataset="deepswe-lean" />\n```');
+ expect(markdown.match(/<EveCodeBenchmark/g)).toHaveLength(1);
+ expect(
+ tree.children.filter(
+ (node) => node.type === "mdxJsxFlowElement" && node.name === "EveCodeBenchmark",
+ ),
+ ).toHaveLength(2);
+ });
+});
diff --git a/apps/docs/lib/geistdocs/remark-benchmark-markdown.ts b/apps/docs/lib/geistdocs/remark-benchmark-markdown.ts
new file mode 100644
index 000000000..bedf1f3a4
--- /dev/null
+++ b/apps/docs/lib/geistdocs/remark-benchmark-markdown.ts
@@ -0,0 +1,25 @@
+import { mdxJsxToMarkdown, type MdxJsxFlowElement } from "mdast-util-mdx-jsx";
+import type { Options } from "mdast-util-to-markdown";
+import { renderBenchmarkMarkdown } from "../evals/eve-code-format";
+import { eveCodeBenchmark } from "../evals/eve-code-results";
+
+// Customize serialization only: HTML must retain the interactive component.
+export function remarkBenchmarkMarkdown(this: { data(): unknown }): void {
+ const data = this.data() as { toMarkdownExtensions?: Options[] };
+ const extensions = (data.toMarkdownExtensions ??= []);
+ const fallback = mdxJsxToMarkdown().handlers!.mdxJsxFlowElement!;
+ extensions.push({
+ handlers: {
+ mdxJsxFlowElement(node: MdxJsxFlowElement, parent, state, info) {
+ if (node.name !== "EveCodeBenchmark") return fallback(node, parent, state, info);
+ const dataset = node.attributes.find(
+ (attribute) => attribute.type === "mdxJsxAttribute" && attribute.name === "dataset",
+ );
+ if (dataset?.type !== "mdxJsxAttribute" || typeof dataset.value !== "string") {
+ throw new Error("EveCodeBenchmark Markdown export requires a literal dataset string.");
+ }
+ return renderBenchmarkMarkdown(eveCodeBenchmark.datasets[dataset.value], dataset.value);
+ },
+ },
+ });
+}
diff --git a/apps/docs/lib/geistdocs/remark-component-markdown.test.ts b/apps/docs/lib/geistdocs/remark-component-markdown.test.ts
index 5a1ae091e..b815e62f7 100644
--- a/apps/docs/lib/geistdocs/remark-component-markdown.test.ts
+++ b/apps/docs/lib/geistdocs/remark-component-markdown.test.ts
@@ -1,6 +1,7 @@
import { createProcessor } from "@mdx-js/mdx";
import { toMarkdown, type Options } from "mdast-util-to-markdown";
import { describe, expect, it } from "vitest";
+import { remarkComponentMarkdown } from "./remark-component-markdown";
import config from "../../source.config";
describe("component Markdown export", () => {
@@ -17,6 +18,8 @@ describe("component Markdown export", () => {
"",
"<Callout>Keep this content.</Callout>",
"",
+ '<EveCodeBenchmark dataset="deepswe-lean" />',
+ "",
"```mdx",
"<AgentRuntimeDiagram />",
"```",
@@ -51,7 +54,12 @@ describe("component Markdown export", () => {
expect(markdown).toContain("**Workspace** — Persistent per-session files");
expect(markdown).toContain("`/workspace`");
expect(markdown).toContain("`from agent/sandbox/workspace/**`");
- expect(markdown).toContain("<Callout>Keep this content.</Callout>");
+ expect(markdown).toContain(
+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",
+ );
+ expect(markdown).toContain("<Callout>");
+ expect(markdown).toContain("Keep this content.");
+ expect(markdown).toContain("</Callout>");
expect(markdown).toContain("```mdx\n<AgentRuntimeDiagram />\n```");
expect(markdown.match(/<AgentRuntimeDiagram \/>/g)).toHaveLength(1);
diff --git a/apps/docs/lib/geistdocs/remark-component-markdown.ts b/apps/docs/lib/geistdocs/remark-component-markdown.ts
index 2abfd1694..0008b2b22 100644
--- a/apps/docs/lib/geistdocs/remark-component-markdown.ts
+++ b/apps/docs/lib/geistdocs/remark-component-markdown.ts
@@ -1,5 +1,7 @@
-import { mdxJsxToMarkdown } from "mdast-util-mdx-jsx";
+import { mdxJsxToMarkdown, type MdxJsxFlowElement } from "mdast-util-mdx-jsx";
import type { Options } from "mdast-util-to-markdown";
+import { renderBenchmarkMarkdown } from "../evals/eve-code-format";
+import { eveCodeBenchmark } from "../evals/eve-code-results";
import { renderAgentRuntimeDiagramMarkdown } from "./agent-runtime-diagram";
// Customize serialization only: the MDX tree must keep the visual component for HTML.
@@ -9,8 +11,17 @@ export function remarkComponentMarkdown(this: { data(): unknown }): void {
const fallback = mdxJsxToMarkdown().handlers!.mdxJsxFlowElement!;
extensions.push({
handlers: {
- mdxJsxFlowElement(node, parent, state, info) {
+ mdxJsxFlowElement(node: MdxJsxFlowElement, parent, state, info) {
if (node.name === "AgentRuntimeDiagram") return renderAgentRuntimeDiagramMarkdown();
+ if (node.name === "EveCodeBenchmark") {
+ const dataset = node.attributes.find(
+ (attribute) => attribute.type === "mdxJsxAttribute" && attribute.name === "dataset",
+ );
+ if (dataset?.type !== "mdxJsxAttribute" || typeof dataset.value !== "string") {
+ throw new Error("EveCodeBenchmark Markdown export requires a literal dataset string.");
+ }
+ return renderBenchmarkMarkdown(eveCodeBenchmark.datasets[dataset.value], dataset.value);
+ }
return fallback(node, parent, state, info);
},
},
diff --git a/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.ts b/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.ts
index 3da209686..683da16b8 100644
--- a/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.ts
+++ b/e2e/fixtures/agent-pending-approval-projection/evals/pending-approval-projection.eval.ts
@@ -1,21 +1,21 @@
import { defineEval } from "eve/evals";
-const INITIAL_TARGET = "005INITIALTARGET";
-const CORRECTED_TARGET = "005CORRECTEDTARGET";
+const INITIAL_TARGET = "Alice";
+const CORRECTED_TARGET = "Bob";
export default defineEval({
tags: ["real-model"],
- description: "A newer user correction supersedes the model-visible pending-approval projection.",
+ description:
+ "An updated plan uses the latest requested target while an earlier action awaits approval.",
async test(t) {
const turn = await t.send(
[
- "I am the original requester and confirm the intended outcome:",
- `set the target to explicit user id ${CORRECTED_TARGET}, not ${INITIAL_TARGET}.`,
- "The correction is authorized. Please revise and submit the plan now.",
+ `The change plan should now target ${CORRECTED_TARGET} instead of ${INITIAL_TARGET}.`,
+ "Please update the plan and send the revised version.",
].join("\n"),
{
clientContext: [
- `Original request: present a change plan targeting user id ${INITIAL_TARGET}.`,
+ `Alice asked for a change plan targeting ${INITIAL_TARGET}.`,
[
"[Pending approvals]",
"The following tool calls are awaiting approval and have not executed:",
diff --git a/packages/eve/scripts/copy-docs.mjs b/packages/eve/scripts/copy-docs.mjs
index 51873bb69..224db8203 100644
--- a/packages/eve/scripts/copy-docs.mjs
+++ b/packages/eve/scripts/copy-docs.mjs
@@ -23,33 +23,15 @@ const page = await readFile(benchmarkPage, "utf8").catch((error) => {
throw error;
});
if (page !== null) {
+ const { renderBenchmarkMarkdown } =
+ await import("../../../apps/docs/lib/evals/eve-code-format.ts");
const snapshot = JSON.parse(
await readFile(join(monorepoRoot, "apps/docs/lib/evals/eve-code-benchmark.json"), "utf8"),
);
await writeFile(
benchmarkPage,
page.replace(/<EveCodeBenchmark dataset="([^"]+)" \/>/gu, (_, dataset) =>
- benchmarkTable(snapshot.datasets[dataset], dataset),
+ renderBenchmarkMarkdown(snapshot.datasets[dataset], dataset),
),
);
}
-
-function benchmarkTable(results, dataset) {
- if (!results) return `No published ${dataset} results yet.`;
- const pct = (value) => `${Math.round(value * 100)}%`;
- const rows = [...results.harnesses]
- .sort(
- (a, b) => b.resolveRate.estimate - a.resolveRate.estimate || a.latencyP50Ms - b.latencyP50Ms,
- )
- .map(
- (h) =>
- `| \`${h.harness}\` | ${pct(h.resolveRate.estimate)} (${h.resolved}/${h.attempts}) | ${pct(h.resolveRate.low)}–${pct(h.resolveRate.high)} | ${(h.latencyP50Ms / 1000).toFixed(1)}s | ${(h.latencyP90Ms / 1000).toFixed(1)}s | ${h.measuredAt?.slice(0, 10) ?? "—"} |`,
- );
- return [
- `${results.tasks} tasks × ${results.attempts} attempts on \`${results.model.id}\`. Snapshot updated ${results.generatedAt.slice(0, 10)}.`,
- "",
- "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",
- "| --- | --- | --- | --- | --- | --- |",
- ...rows,
- ].join("\n");
-}