Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

1.3 KB
Raw Blame
import type { EveCodeDatasetResults, EveCodeHarnessScore } from "./eve-code-results";

/** Ranks by resolve rate, then by median latency, matching eve-bench's correctness-first order. */
export function rankedHarnesses(results: EveCodeDatasetResults): EveCodeHarnessScore[] {
  return [...results.harnesses].sort(
    (left, right) =>
      right.resolveRate.estimate - left.resolveRate.estimate ||
      left.latencyP50Ms - right.latencyP50Ms,
  );
}

export function renderBenchmarkMarkdown(
  results: EveCodeDatasetResults | undefined,
  dataset: string,
): string {
  if (!results) return `No published ${dataset} results yet.`;
  const pct = (value: number) => `${Math.round(value * 100)}%`;
  const rows = rankedHarnesses(results).map(
    (h) =>
      `| \`${h.harness}\` | ${pct(h.resolveRate.estimate)} (${h.resolved}/${h.attempts}) | ${pct(h.resolveRate.low)}–${pct(h.resolveRate.high)} | ${(h.latencyP50Ms / 1000).toFixed(1)}s | ${(h.latencyP90Ms / 1000).toFixed(1)}s | ${h.measuredAt?.slice(0, 10) ?? "—"} |`,
  );
  return [
    `${results.tasks} tasks × ${results.attempts} attempts on \`${results.model.id}\`. Snapshot updated ${results.generatedAt.slice(0, 10)}.`,
    "",
    "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",
    "| --- | --- | --- | --- | --- | --- |",
    ...rows,
  ].join("\n");
}