Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

docs(benchmarks) anthropic to use claude harness introduce opus 5 (#2731)

Colton Padden83d40081dd7c
diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjs
index e6d68c63b..dc550aafa 100644
--- a/apps/benchmarks/lib/benchmark-config.test.mjs
+++ b/apps/benchmarks/lib/benchmark-config.test.mjs
@@ -19,6 +19,7 @@ test("publishes only compatibility-validated models", () => {
"gpt-5-6-terra",
"claude-sonnet-5",
"glm-5-2",
+ "claude-opus-5",
"gemini-3-1-pro-preview",
],
);
@@ -34,6 +35,10 @@ test("publishes only compatibility-validated models", () => {
assert.equal(findPublishedBenchmarkModel("grok-4-6").model, "xai/grok-4.6");
assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").model, "anthropic/claude-sonnet-5");
assert.equal(findPublishedBenchmarkModel("glm-5-2").model, "zai/glm-5.2");
+ assert.equal(findPublishedBenchmarkModel("claude-fable-5").harness, "Claude Code");
+ assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code");
+ assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code");
+ assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode");
});
test("allows candidate probes and rejects unknown models", () => {
@@ -41,6 +46,6 @@ test("allows candidate probes and rejects unknown models", () => {
assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");
assert.equal(findBenchmarkModel("grok-4-6").support, "supported");
assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");
- assert.equal(findBenchmarkModel("claude-opus-5").support, "candidate");
+ assert.equal(findBenchmarkModel("claude-opus-5").support, "supported");
assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);
});
diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.ts
index 3abb3cfbc..eb2185b75 100644
--- a/apps/benchmarks/lib/benchmark-config.ts
+++ b/apps/benchmarks/lib/benchmark-config.ts
@@ -8,7 +8,7 @@ export interface AuthoringBenchmarkModel {
readonly id: string;
readonly model: string;
readonly displayName: string;
- readonly harness: "OpenCode";
+ readonly harness: "OpenCode" | "Claude Code";
readonly support: AuthoringBenchmarkSupport;
}
@@ -31,7 +31,7 @@ export const benchmarkModels = [
id: "claude-fable-5",
model: "anthropic/claude-fable-5",
displayName: "Claude Fable 5",
- harness: "OpenCode",
+ harness: "Claude Code",
support: "supported",
},
{
@@ -59,7 +59,7 @@ export const benchmarkModels = [
id: "claude-sonnet-5",
model: "anthropic/claude-sonnet-5",
displayName: "Claude Sonnet 5",
- harness: "OpenCode",
+ harness: "Claude Code",
support: "supported",
},
{
@@ -73,8 +73,8 @@ export const benchmarkModels = [
id: "claude-opus-5",
model: "anthropic/claude-opus-5",
displayName: "Claude Opus 5",
- harness: "OpenCode",
- support: "candidate",
+ harness: "Claude Code",
+ support: "supported",
},
{
id: "gemini-3-1-pro-preview",
@@ -103,10 +103,14 @@ export const publishedBenchmark = {
} as const;
export function publishedExperimentId(
- benchmark: Pick<AuthoringBenchmarkModel, "id">,
+ benchmark: Pick<AuthoringBenchmarkModel, "id" | "harness">,
treatment: AuthoringTreatment,
): string {
- return `${benchmark.id}-opencode--${treatment}`;
+ return `${benchmark.id}-${harnessId(benchmark.harness)}--${treatment}`;
+}
+
+export function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {
+ return harness === "Claude Code" ? "claude-code" : "opencode";
}
export function parseAuthoringTreatment(value: string): AuthoringTreatment {
diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.ts
index f18c9a4bc..71e6f8bdb 100644
--- a/apps/benchmarks/lib/experiment.ts
+++ b/apps/benchmarks/lib/experiment.ts
@@ -16,6 +16,7 @@ export function authoringExperiment(options: {
readonly verbose?: boolean;
}): ExperimentConfig {
const agent = createAuthoringAgent({
+ harness: options.benchmark.harness,
model: options.benchmark.model,
archive: options.archive,
dependencyArchive: options.dependencyArchive,
diff --git a/apps/benchmarks/lib/harness-agent.test.mjs b/apps/benchmarks/lib/harness-agent.test.mjs
new file mode 100644
index 000000000..c49a19937
--- /dev/null
+++ b/apps/benchmarks/lib/harness-agent.test.mjs
@@ -0,0 +1,56 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { createJiti } from "jiti";
+
+const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });
+const { addUsage } = await jiti.import(new URL("./harness-agent.ts", import.meta.url).pathname);
+
+test("normalizes nested Claude Code token usage", () => {
+ const usage = {
+ inputTokens: 0,
+ outputTokens: 0,
+ reasoningTokens: 0,
+ cachedInputTokens: 0,
+ cacheWriteTokens: 0,
+ };
+
+ addUsage(usage, {
+ inputTokens: { total: 32_177, noCache: 1, cacheRead: 32_176, cacheWrite: 10_944 },
+ outputTokens: { total: 327, text: 327 },
+ });
+
+ assert.deepEqual(usage, {
+ inputTokens: 1,
+ outputTokens: 327,
+ reasoningTokens: 0,
+ cachedInputTokens: 32_176,
+ cacheWriteTokens: 10_944,
+ });
+});
+
+test("retains flat OpenCode token usage", () => {
+ const usage = {
+ inputTokens: 0,
+ outputTokens: 0,
+ reasoningTokens: 0,
+ cachedInputTokens: 0,
+ cacheWriteTokens: 0,
+ };
+
+ addUsage(usage, {
+ inputTokens: 100,
+ outputTokens: 20,
+ reasoningTokens: 5,
+ cachedInputTokens: 50,
+ cacheWriteTokens: 10,
+ });
+
+ assert.deepEqual(usage, {
+ inputTokens: 100,
+ outputTokens: 20,
+ reasoningTokens: 5,
+ cachedInputTokens: 50,
+ cacheWriteTokens: 10,
+ });
+});
diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.ts
index c2feb69a8..eacbb47f0 100644
--- a/apps/benchmarks/lib/harness-agent.ts
+++ b/apps/benchmarks/lib/harness-agent.ts
@@ -1,7 +1,8 @@
import { readFileSync } from "node:fs";
-import type { HarnessV1NetworkSandboxSession } from "@ai-sdk/harness";
+import type { HarnessV1, HarnessV1NetworkSandboxSession } from "@ai-sdk/harness";
import type { HarnessAgentSession } from "@ai-sdk/harness/agent";
import { HarnessAgent } from "@ai-sdk/harness/agent";
+import { createClaudeCode } from "@ai-sdk/harness-claude-code";
import { createOpenCode } from "@ai-sdk/harness-opencode";
import type { Agent, AgentRunResult } from "@vercel/agent-eval";
import { z } from "zod";
@@ -12,6 +13,7 @@ import type {
AuthoringSetupContext,
AuthoringTurn,
} from "./authoring-case.js";
+import type { AuthoringBenchmarkModel } from "./benchmark-config.js";
import { createDependencyCachedSandbox } from "./dependency-sandbox.js";
import { loadAuthoringCase } from "./load-authoring-case.js";
import {
@@ -28,15 +30,15 @@ import { BenchmarkTimings } from "./timing.js";
const HARNESS_BRIDGE_PORT = 4172;
const POST_RUN_GRADER_DIRECTORY = ".eve-grader";
-const BOOTSTRAP_VERSION = "v8";
+const BOOTSTRAP_VERSION = "v9";
// A turn that stops producing output should end the turn, not the eval: the
// remaining turns still run and the graders still see what the agent did.
-const TURN_TIMEOUT_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_TIMEOUT ?? 480);
+const TURN_TIMEOUT_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_TIMEOUT ?? 300);
// A turn that has produced no chunk for this long is waiting on the runtime, not
// working. The bound is one slow tool call: the bridge reports a call and its
// result together once the call returns, so an install that takes minutes looks
// from here like silence rather than work in flight.
-const TURN_STALL_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_STALL ?? 240);
+const TURN_STALL_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_STALL ?? 120);
// The runtime frequently never closes a turn whose last act was the model
// talking, so silence after a closing message has to end the turn on its own.
// Long enough that a pause before the next tool call is not mistaken for one.
@@ -49,10 +51,7 @@ const TURN_SETTLE_MILLIS = 15_000;
// the turn, so the harness answers it immediately and points the agent at the
// channel this benchmark can answer on: its reply, which the case's next `send`
// responds to.
-type AuthoringHarnessAgent = HarnessAgent<
- ReturnType<typeof createOpenCode>,
- typeof INTERACTIVE_QUESTION_TOOL
->;
+type AuthoringHarnessAgent = HarnessAgent<HarnessV1, typeof INTERACTIVE_QUESTION_TOOL>;
const INTERACTIVE_QUESTION_TOOL = {
question: {
@@ -64,6 +63,7 @@ const INTERACTIVE_QUESTION_TOOL = {
};
export function createAuthoringAgent(subject: {
+ readonly harness: AuthoringBenchmarkModel["harness"];
readonly model: string;
readonly archive: Uint8Array;
readonly dependencyArchive: Uint8Array;
@@ -71,15 +71,15 @@ export function createAuthoringAgent(subject: {
readonly dependencyDigest: string;
}): Agent {
return {
- name: "opencode",
- displayName: "eve authoring harness",
+ name: harnessId(subject.harness),
+ displayName: `${subject.harness} eve authoring harness`,
getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY",
getDefaultModel: () => subject.model,
definition: {
- name: "opencode",
- displayName: "eve authoring harness",
+ name: harnessId(subject.harness),
+ displayName: `${subject.harness} eve authoring harness`,
defaultModel: subject.model,
- o11yAgentName: "opencode",
+ o11yAgentName: harnessId(subject.harness),
runnerPath: "",
getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY",
install: () => [],
@@ -126,11 +126,7 @@ export function createAuthoringAgent(subject: {
const agent = new HarnessAgent({
id: "eve-authoring-eval",
- harness: createOpenCode({
- auth: "ai-gateway",
- model: openCodeModel(options.model ?? subject.model),
- port: HARNESS_BRIDGE_PORT,
- }),
+ harness: createHarness(subject.harness, options.model ?? subject.model),
sandbox,
tools: INTERACTIVE_QUESTION_TOOL,
sandboxConfig: {
@@ -159,6 +155,12 @@ export function createAuthoringAgent(subject: {
activeSandbox = current as HarnessV1NetworkSandboxSession;
workspace = sessionWorkDir;
const context = setupContext(activeSandbox, workspace);
+ await ensureWorkspace(
+ context,
+ authoringCase.startingPoint.workspace,
+ subject.archive,
+ timings,
+ );
await timings.measure("session.setup", async () => {
for (const setup of setups) await setup.onSession?.(context);
if (options.agentOptions?.agentsMd !== true) await installBaselineEveWrapper(context);
@@ -168,15 +170,6 @@ export function createAuthoringAgent(subject: {
}
},
},
- // OpenCode subagents run outside the observed harness session: their tool
- // calls and token usage are not captured, and the pinned adapter can hang
- // waiting for the delegated task to return. Keep every model in one
- // session using direct filesystem and shell tools so runs stay observable
- // and comparable. Ending promptly after implementation also prevents a
- // completed change from becoming an infrastructure timeout while the
- // model continues unrequested exploration or validation.
- instructions:
- "Work directly in the current session. Do not delegate to subagents; use the available filesystem and shell tools yourself. Once the requested implementation is complete, stop promptly without additional exploration or validation unless the user requested it.",
permissionMode: "allow-all",
});
@@ -192,9 +185,9 @@ export function createAuthoringAgent(subject: {
session,
transcript,
send: async (prompt) => {
+ const turn = transcript.filter((entry) => entry.role === "user").length + 1;
transcript.push({ role: "user", content: prompt });
if (verbose) console.log(`[user] ${prompt}`);
- const turn = transcript.filter((entry) => entry.role === "user").length;
const result = await timings.measure(`agent.turn.${turn}`, () =>
runTurn({
agent,
@@ -446,6 +439,7 @@ async function runTurn(input: {
turnLooksDone = !stepCalledTool;
stepCalledTool = false;
} else if (part.type === "finish") {
+ replaceUsage(usage, (part as { totalUsage?: unknown }).totalUsage);
turnLooksDone = true;
}
}
@@ -502,13 +496,60 @@ async function withDeadline<T>(promise: Promise<T>, millis: number): Promise<T |
}
}
-function addUsage(total: Record<string, number>, value: unknown): void {
+type MutableAuthoringTokenUsage = {
+ -readonly [Key in keyof AuthoringTokenUsage]: AuthoringTokenUsage[Key];
+};
+
+export function addUsage(total: MutableAuthoringTokenUsage, value: unknown): void {
if (typeof value !== "object" || value === null) return;
- for (const [field, amount] of Object.entries(value as Record<string, unknown>)) {
- if (typeof amount === "number" && Number.isFinite(amount) && total[field] !== undefined) {
- total[field] += amount;
- }
+ const usage = value as Record<string, unknown>;
+ const input = usage.inputTokens;
+ const output = usage.outputTokens;
+ if (typeof input === "object" && input !== null) {
+ const tokens = input as Record<string, unknown>;
+ total.inputTokens += usageNumber(tokens.noCache ?? tokens.total);
+ total.cachedInputTokens += usageNumber(tokens.cacheRead);
+ total.cacheWriteTokens += usageNumber(tokens.cacheWrite);
+ } else {
+ total.inputTokens += usageNumber(input);
+ }
+ if (typeof output === "object" && output !== null) {
+ total.outputTokens += usageNumber((output as Record<string, unknown>).total);
+ } else {
+ total.outputTokens += usageNumber(output);
+ }
+ total.reasoningTokens += usageNumber(usage.reasoningTokens);
+ total.cachedInputTokens += usageNumber(usage.cachedInputTokens);
+ total.cacheWriteTokens += usageNumber(usage.cacheWriteTokens);
+}
+
+function replaceUsage(total: MutableAuthoringTokenUsage, value: unknown): void {
+ if (typeof value !== "object" || value === null) return;
+ total.inputTokens = 0;
+ total.outputTokens = 0;
+ total.reasoningTokens = 0;
+ total.cachedInputTokens = 0;
+ total.cacheWriteTokens = 0;
+ addUsage(total, value);
+}
+
+function usageNumber(value: unknown): number {
+ return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+function createHarness(harness: AuthoringBenchmarkModel["harness"], model: string): HarnessV1 {
+ if (harness === "Claude Code") {
+ return createClaudeCode({ auth: { gateway: {} }, model });
}
+ return createOpenCode({
+ auth: "ai-gateway",
+ model: openCodeModel(model),
+ port: HARNESS_BRIDGE_PORT,
+ });
+}
+
+function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {
+ return harness === "Claude Code" ? "claude-code" : "opencode";
}
function openCodeModel(model: string): string {
@@ -555,15 +596,8 @@ async function bootstrapSubject(
if (workspaceKind === "scaffolded") {
workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`);
}
- // Grader files must not change what subject commands observe in the project directory.
- workspaceCommands.push(
- `rm -rf ${AGENT_EVAL_DIRECTORY} && mkdir -p ${AGENT_EVAL_DIRECTORY}`,
- `printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,
- "command -v vitest >/dev/null",
- );
- if (workspaceKind === "scaffolded") {
- workspaceCommands.push(`test -f ${workspace}/package.json`);
- }
+ // Grader files are created after the agent finishes so an empty starting point remains empty.
+ workspaceCommands.push("command -v vitest >/dev/null");
await timings.measure("subject.workspace-bootstrap", () =>
run(sandbox, workspaceCommands.join(" && ")),
);
@@ -592,6 +626,44 @@ function setupContext(
// Created only after the agent's turns finish: an `empty` starting point has to
// look empty to `eve init .`, which refuses to scaffold into a directory that
// already holds entries it does not recognize.
+async function ensureWorkspace(
+ context: AuthoringSetupContext,
+ workspaceKind: "scaffolded" | "empty",
+ archive: Uint8Array,
+ timings: BenchmarkTimings,
+): Promise<void> {
+ if (workspaceKind === "empty") {
+ await verifyWorkspace(context, workspaceKind);
+ return;
+ }
+ if (await hasPreparedWorkspace(context)) return;
+
+ await timings.measure("session.workspace-recovery", () =>
+ bootstrapSubject(context.sandbox, context.workspace, workspaceKind, archive, timings),
+ );
+ await verifyWorkspace(context, workspaceKind);
+}
+
+async function hasPreparedWorkspace(context: AuthoringSetupContext): Promise<boolean> {
+ const result = await resultOf(
+ context.sandbox,
+ `test -f package.json && test -f agent/instructions.md && test -f AGENTS.md && test -f ${EVE_PACKAGE_PATH}`,
+ context.workspace,
+ );
+ return result.exitCode === 0;
+}
+
+async function verifyWorkspace(
+ context: AuthoringSetupContext,
+ workspaceKind: "scaffolded" | "empty",
+): Promise<void> {
+ const command =
+ workspaceKind === "empty"
+ ? "test ! -e package.json && test ! -e AGENTS.md && test ! -e agent && test ! -e .eve-grader"
+ : `for path in package.json agent/instructions.md AGENTS.md ${EVE_PACKAGE_PATH}; do test -f "$path" || { echo "Missing prepared workspace file: $path" >&2; exit 1; }; done`;
+ await context.run(command);
+}
+
async function prepareGraderDirectory(context: AuthoringSetupContext): Promise<void> {
await context.run(
`mkdir -p ${AGENT_EVAL_DIRECTORY} && printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,
diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.json
index 57b72d598..348170ae9 100644
--- a/apps/benchmarks/package.json
+++ b/apps/benchmarks/package.json
@@ -12,11 +12,12 @@
"benchmark:timings": "node scripts/timings.mjs"
},
"dependencies": {
- "@ai-sdk/harness": "1.0.72",
- "@ai-sdk/harness-opencode": "1.0.73",
- "@ai-sdk/sandbox-vercel": "1.0.72",
+ "@ai-sdk/harness": "1.0.87",
+ "@ai-sdk/harness-claude-code": "1.0.90",
+ "@ai-sdk/harness-opencode": "1.0.88",
+ "@ai-sdk/sandbox-vercel": "1.0.87",
"@vercel/agent-eval": "1.5.0",
- "@vercel/sandbox": "catalog:",
+ "@vercel/sandbox": "3.2.0",
"jiti": "2.7.0",
"zod": "catalog:"
},
diff --git a/apps/benchmarks/scripts/cost.mjs b/apps/benchmarks/scripts/cost.mjs
index 66119b0fd..3a4e840c3 100644
--- a/apps/benchmarks/scripts/cost.mjs
+++ b/apps/benchmarks/scripts/cost.mjs
@@ -49,6 +49,14 @@ export const modelPricing = {
cacheRead: 0.2,
cacheWrite: 2.5,
},
+ "anthropic/claude-opus-5": {
+ effectiveDate: "2026-08-28",
+ source: "https://models.dev/anthropic/claude-opus-5",
+ input: 5,
+ output: 25,
+ cacheRead: 0.5,
+ cacheWrite: 6.25,
+ },
"anthropic/claude-sonnet-5": {
effectiveDate: "2026-08-17",
source: "https://docs.anthropic.com/en/docs/about-claude/pricing",
@@ -97,7 +105,7 @@ export function extractRunUsage(raw) {
// Ignore malformed transcript lines.
}
}
- return found ? usage : null;
+ return found && tokenConsumption(usage) > 0 ? usage : null;
}
export function tokenConsumption(usage) {
diff --git a/apps/benchmarks/scripts/export-results.mjs b/apps/benchmarks/scripts/export-results.mjs
index 0d1c71e5b..bf1e4f505 100644
--- a/apps/benchmarks/scripts/export-results.mjs
+++ b/apps/benchmarks/scripts/export-results.mjs
@@ -21,6 +21,7 @@ import {
publishedBenchmark,
publishedBenchmarkModels,
publishedExperimentId,
+ harnessId,
} from "../lib/benchmark-config.ts";
const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
@@ -98,7 +99,7 @@ const output = {
experiments: benchmarks.flatMap((benchmark) =>
authoringTreatments.map((treatment) => ({
id: publishedExperimentId(benchmark, treatment),
- groupId: `${benchmark.id}-opencode`,
+ groupId: `${benchmark.id}-${harnessId(benchmark.harness)}`,
model: benchmark.model,
modelDisplayName: benchmark.displayName,
harness: benchmark.harness,
@@ -131,12 +132,26 @@ function readPreviousResults() {
const supersededExperiments = (previous.experiments ?? []).filter((experiment) =>
supersededExperimentIds.has(experiment.id),
);
+ const archivedExperimentIds = new Set(supersededExperiments.map((experiment) => experiment.id));
const supersededResults = (previous.results ?? []).filter((result) =>
- supersededExperimentIds.has(result.experimentId),
- );
- const retained = (previous.previouslyMeasured ?? []).filter(
- (measurement) => Array.isArray(measurement.experiments) && Array.isArray(measurement.results),
+ archivedExperimentIds.has(result.experimentId),
);
+ const retained = (previous.previouslyMeasured ?? [])
+ .filter(
+ (measurement) => Array.isArray(measurement.experiments) && Array.isArray(measurement.results),
+ )
+ .map((measurement) => {
+ const experiments = measurement.experiments.filter((experiment) =>
+ supersededExperimentIds.has(experiment.id),
+ );
+ const experimentIds = new Set(experiments.map((experiment) => experiment.id));
+ return {
+ ...measurement,
+ experiments,
+ results: measurement.results.filter((result) => experimentIds.has(result.experimentId)),
+ };
+ })
+ .filter((measurement) => measurement.experiments.length > 0);
if (supersededExperiments.length === 0) return retained;
return [
@@ -213,7 +228,28 @@ function latestValidResult(experimentId, caseId) {
}
function meanRunMetrics(summaryPath, model) {
- const runs = readdirSync(dirname(summaryPath), { withFileTypes: true })
+ const performanceRuns = runMetrics(summaryPath);
+ const performanceUsage = performanceRuns.flatMap((run) =>
+ run.usage === null ? [] : [run.usage],
+ );
+ const result = {};
+ const pricing = modelPricing[model];
+ if (pricing !== undefined && performanceUsage.length > 0) {
+ result.meanEstimatedListCostUsd = mean(
+ performanceUsage.map((value) => priceUsage(value, pricing)),
+ );
+ }
+ if (performanceUsage.length > 0) {
+ result.meanTokenConsumption = mean(performanceUsage.map(tokenConsumption));
+ }
+ if (performanceRuns.length > 0) {
+ result.meanToolInvocationCount = mean(performanceRuns.map((run) => run.toolInvocations));
+ }
+ return result;
+}
+
+function runMetrics(summaryPath) {
+ return readdirSync(dirname(summaryPath), { withFileTypes: true })
.filter((entry) => entry.isDirectory() && /^run-\d+$/u.test(entry.name))
.flatMap((entry) => {
const transcriptPath = join(dirname(summaryPath), entry.name, "transcript-raw.jsonl");
@@ -221,19 +257,6 @@ function meanRunMetrics(summaryPath, model) {
const raw = readFileSync(transcriptPath, "utf8");
return [{ usage: extractRunUsage(raw), toolInvocations: countToolInvocations(raw) }];
});
- const usage = runs.flatMap((run) => (run.usage === null ? [] : [run.usage]));
- const result = {};
- const pricing = modelPricing[model];
- if (pricing !== undefined && usage.length > 0) {
- result.meanEstimatedListCostUsd = mean(usage.map((value) => priceUsage(value, pricing)));
- }
- if (usage.length > 0) {
- result.meanTokenConsumption = mean(usage.map(tokenConsumption));
- }
- if (runs.length > 0) {
- result.meanToolInvocationCount = mean(runs.map((run) => run.toolInvocations));
- }
- return result;
}
function mean(values) {
diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjs
index 0d19c8aaa..e96c56d98 100644
--- a/apps/benchmarks/scripts/export-results.test.mjs
+++ b/apps/benchmarks/scripts/export-results.test.mjs
@@ -40,7 +40,7 @@ test("exports missing cells without publishing private artifacts", () => {
assert.equal(output.schemaVersion, 1);
assert.equal(output.suite.caseCount, 7);
assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);
- assert.equal(output.experiments.length, 16);
+ assert.equal(output.experiments.length, 18);
assert.deepEqual(
[...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],
[
@@ -51,10 +51,11 @@ test("exports missing cells without publishing private artifacts", () => {
"GPT-5.6 Terra",
"Claude Sonnet 5",
"GLM 5.2",
+ "Claude Opus 5",
"Gemini 3.1 Pro Preview",
],
);
- assert.equal(output.results.length, 112);
+ assert.equal(output.results.length, 126);
assert.ok(output.results.every((result) => result.status === "missing"));
for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {
assert.equal(raw.includes(`"${privateField}"`), false);
diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.json
index 312ba679a..03b21ac82 100644
--- a/apps/docs/lib/evals/benchmark-results.json
+++ b/apps/docs/lib/evals/benchmark-results.json
@@ -1,6 +1,6 @@
{
"schemaVersion": 1,
- "generatedAt": "2026-08-28T20:19:42.291Z",
+ "generatedAt": "2026-08-29T02:09:15.189Z",
"suite": {
"eveRevision": "05b12e3beb662979143397b9e691a2d7b4417a02",
"caseFingerprint": "786490be9d902701f57d08b3d112ca385d49a72e5591fac741511e77d67989d2",
@@ -25,19 +25,19 @@
"treatment": "guided"
},
{
- "id": "claude-fable-5-opencode--baseline",
- "groupId": "claude-fable-5-opencode",
+ "id": "claude-fable-5-claude-code--baseline",
+ "groupId": "claude-fable-5-claude-code",
"model": "anthropic/claude-fable-5",
"modelDisplayName": "Claude Fable 5",
- "harness": "OpenCode",
+ "harness": "Claude Code",
"treatment": "baseline"
},
{
- "id": "claude-fable-5-opencode--guided",
- "groupId": "claude-fable-5-opencode",
+ "id": "claude-fable-5-claude-code--guided",
+ "groupId": "claude-fable-5-claude-code",
"model": "anthropic/claude-fable-5",
"modelDisplayName": "Claude Fable 5",
- "harness": "OpenCode",
+ "harness": "Claude Code",
"treatment": "guided"
},
{
@@ -89,19 +89,19 @@
"treatment": "guided"
},
{
- "id": "claude-sonnet-5-opencode--baseline",
- "groupId": "claude-sonnet-5-opencode",
+ "id": "claude-sonnet-5-claude-code--baseline",
+ "groupId": "claude-sonnet-5-claude-code",
"model": "anthropic/claude-sonnet-5",
"modelDisplayName": "Claude Sonnet 5",
- "harness": "OpenCode",
+ "harness": "Claude Code",
"treatment": "baseline"
},
{
- "id": "claude-sonnet-5-opencode--guided",
- "groupId": "claude-sonnet-5-opencode",
+ "id": "claude-sonnet-5-claude-code--guided",
+ "groupId": "claude-sonnet-5-claude-code",
"model": "anthropic/claude-sonnet-5",
"modelDisplayName": "Claude Sonnet 5",
- "harness": "OpenCode",
+ "harness": "Claude Code",
"treatment": "guided"
},
{
@@ -120,6 +120,22 @@
"harness": "OpenCode",
"treatment": "guided"
},
+ {
+ "id": "claude-opus-5-claude-code--baseline",
+ "groupId": "claude-opus-5-claude-code",
+ "model": "anthropic/claude-opus-5",
+ "modelDisplayName": "Claude Opus 5",
+ "harness": "Claude Code",
+ "treatment": "baseline"
+ },
+ {
+ "id": "claude-opus-5-claude-code--guided",
+ "groupId": "claude-opus-5-claude-code",
+ "model": "anthropic/claude-opus-5",
+ "modelDisplayName": "Claude Opus 5",
+ "harness": "Claude Code",
+ "treatment": "guided"
+ },
{
"id": "gemini-3-1-pro-preview-opencode--baseline",
"groupId": "gemini-3-1-pro-preview-opencode",
@@ -307,172 +323,172 @@
"meanToolInvocationCount": 12.333333333333334
},
{
- "experimentId": "claude-fable-5-opencode--baseline",
+ "experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 72217,
- "measuredAt": "2026-08-28T19:18:44.000Z",
- "meanEstimatedListCostUsd": 0.30839,
- "meanTokenConsumption": 19988.333333333332,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 127045,
+ "measuredAt": "2026-08-28T22:06:23.308Z",
+ "meanEstimatedListCostUsd": 6.18823,
+ "meanToolInvocationCount": 17.666666666666668,
+ "meanTokenConsumption": 598539
},
{
- "experimentId": "claude-fable-5-opencode--baseline",
+ "experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 93899,
- "measuredAt": "2026-08-28T19:25:21.000Z",
- "meanEstimatedListCostUsd": 0.22944333333333336,
- "meanTokenConsumption": 17757.666666666668,
- "meanToolInvocationCount": 9
+ "meanDurationMs": 169780,
+ "measuredAt": "2026-08-28T22:06:23.319Z",
+ "meanEstimatedListCostUsd": 3.83675,
+ "meanToolInvocationCount": 8.666666666666666,
+ "meanTokenConsumption": 375135
},
{
- "experimentId": "claude-fable-5-opencode--baseline",
+ "experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 80987,
- "measuredAt": "2026-08-28T19:25:21.000Z",
- "meanEstimatedListCostUsd": 0.3146933333333333,
- "meanTokenConsumption": 23109.333333333332,
- "meanToolInvocationCount": 16.666666666666668
+ "meanDurationMs": 258551,
+ "measuredAt": "2026-08-28T22:06:23.327Z",
+ "meanEstimatedListCostUsd": 5.43717,
+ "meanToolInvocationCount": 12.666666666666666,
+ "meanTokenConsumption": 528873
},
{
- "experimentId": "claude-fable-5-opencode--baseline",
+ "experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 62158,
- "measuredAt": "2026-08-28T19:18:44.000Z",
- "meanEstimatedListCostUsd": 0.22673666666666667,
- "meanTokenConsumption": 14869.666666666666,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 89365,
+ "measuredAt": "2026-08-28T22:06:23.334Z",
+ "meanEstimatedListCostUsd": 3.97663,
+ "meanToolInvocationCount": 11.666666666666666,
+ "meanTokenConsumption": 383479
},
{
- "experimentId": "claude-fable-5-opencode--baseline",
+ "experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 55663,
- "measuredAt": "2026-08-28T19:18:44.000Z",
- "meanEstimatedListCostUsd": 0.30195666666666665,
- "meanTokenConsumption": 24389,
- "meanToolInvocationCount": 11.666666666666666
+ "meanDurationMs": 123612,
+ "measuredAt": "2026-08-28T22:06:23.339Z",
+ "meanEstimatedListCostUsd": 5.19217,
+ "meanToolInvocationCount": 17.666666666666668,
+ "meanTokenConsumption": 507153
},
{
- "experimentId": "claude-fable-5-opencode--baseline",
+ "experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 53720,
- "measuredAt": "2026-08-28T19:18:44.000Z",
- "meanEstimatedListCostUsd": 0.2800366666666667,
- "meanTokenConsumption": 23315.666666666668,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 104483,
+ "measuredAt": "2026-08-28T22:06:23.346Z",
+ "meanEstimatedListCostUsd": 7.76357,
+ "meanToolInvocationCount": 15.666666666666666,
+ "meanTokenConsumption": 761257
},
{
- "experimentId": "claude-fable-5-opencode--baseline",
+ "experimentId": "claude-fable-5-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 53286,
- "measuredAt": "2026-08-28T19:18:44.000Z",
- "meanEstimatedListCostUsd": 0.2505233333333334,
- "meanTokenConsumption": 19769.666666666668,
- "meanToolInvocationCount": 11
+ "meanDurationMs": 156463,
+ "measuredAt": "2026-08-28T22:06:23.352Z",
+ "meanEstimatedListCostUsd": 3.47642,
+ "meanToolInvocationCount": 23.666666666666668,
+ "meanTokenConsumption": 339290
},
{
- "experimentId": "claude-fable-5-opencode--guided",
+ "experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 53166,
- "measuredAt": "2026-08-28T19:18:27.000Z",
- "meanEstimatedListCostUsd": 0.3223233333333333,
- "meanTokenConsumption": 23616.333333333332,
- "meanToolInvocationCount": 7.333333333333333
+ "meanDurationMs": 80508,
+ "measuredAt": "2026-08-28T22:36:19.362Z",
+ "meanEstimatedListCostUsd": 3.87527,
+ "meanToolInvocationCount": 7.666666666666667,
+ "meanTokenConsumption": 375115
},
{
- "experimentId": "claude-fable-5-opencode--guided",
+ "experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 85836,
- "measuredAt": "2026-08-28T19:25:03.000Z",
- "meanEstimatedListCostUsd": 0.24718666666666667,
- "meanTokenConsumption": 19625.333333333332,
- "meanToolInvocationCount": 8.333333333333334
+ "meanDurationMs": 80206,
+ "measuredAt": "2026-08-28T22:36:19.376Z",
+ "meanEstimatedListCostUsd": 3.40311,
+ "meanToolInvocationCount": 9,
+ "meanTokenConsumption": 332259
},
{
- "experimentId": "claude-fable-5-opencode--guided",
+ "experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
- "passedRuns": 0,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 277471,
- "measuredAt": "2026-08-28T19:18:27.000Z",
- "meanEstimatedListCostUsd": 0.4114,
- "meanTokenConsumption": 19754.666666666668,
- "meanToolInvocationCount": 7.666666666666667
+ "meanDurationMs": 71462,
+ "measuredAt": "2026-08-28T22:36:19.386Z",
+ "meanEstimatedListCostUsd": 3.58212,
+ "meanToolInvocationCount": 8.666666666666666,
+ "meanTokenConsumption": 350688
},
{
- "experimentId": "claude-fable-5-opencode--guided",
+ "experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 70438,
- "measuredAt": "2026-08-28T19:18:27.000Z",
- "meanEstimatedListCostUsd": 0.25587,
- "meanTokenConsumption": 16792.333333333332,
- "meanToolInvocationCount": 11.333333333333334
+ "meanDurationMs": 87034,
+ "measuredAt": "2026-08-28T22:36:19.391Z",
+ "meanEstimatedListCostUsd": 4.10959,
+ "meanToolInvocationCount": 7.666666666666667,
+ "meanTokenConsumption": 397447
},
{
- "experimentId": "claude-fable-5-opencode--guided",
+ "experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 41451,
- "measuredAt": "2026-08-28T19:18:27.000Z",
- "meanEstimatedListCostUsd": 0.29036,
- "meanTokenConsumption": 24874.666666666668,
- "meanToolInvocationCount": 7
+ "meanDurationMs": 59604,
+ "measuredAt": "2026-08-28T22:36:19.396Z",
+ "meanEstimatedListCostUsd": 3.38543,
+ "meanToolInvocationCount": 7,
+ "meanTokenConsumption": 331919
},
{
- "experimentId": "claude-fable-5-opencode--guided",
+ "experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 39958,
- "measuredAt": "2026-08-28T19:18:27.000Z",
- "meanEstimatedListCostUsd": 0.21587,
- "meanTokenConsumption": 18063,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 64829,
+ "measuredAt": "2026-08-28T22:36:19.402Z",
+ "meanEstimatedListCostUsd": 3.81826,
+ "meanToolInvocationCount": 7,
+ "meanTokenConsumption": 374610
},
{
- "experimentId": "claude-fable-5-opencode--guided",
+ "experimentId": "claude-fable-5-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 36170,
- "measuredAt": "2026-08-28T19:18:27.000Z",
- "meanEstimatedListCostUsd": 0.16552,
- "meanTokenConsumption": 13610.666666666666,
- "meanToolInvocationCount": 6.333333333333333
+ "meanDurationMs": 71055,
+ "measuredAt": "2026-08-28T22:36:19.407Z",
+ "meanEstimatedListCostUsd": 3.07521,
+ "meanToolInvocationCount": 7,
+ "meanTokenConsumption": 299897
},
{
"experimentId": "grok-4-6-opencode--baseline",
@@ -979,172 +995,172 @@
"meanToolInvocationCount": 11.666666666666666
},
{
- "experimentId": "claude-sonnet-5-opencode--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 84504,
- "measuredAt": "2026-08-28T19:20:33.000Z",
- "meanEstimatedListCostUsd": 0.087304,
- "meanTokenConsumption": 30901.333333333332,
- "meanToolInvocationCount": 17.666666666666668
+ "meanDurationMs": 105859,
+ "measuredAt": "2026-08-28T22:44:03.383Z",
+ "meanEstimatedListCostUsd": 1.735734,
+ "meanToolInvocationCount": 18,
+ "meanTokenConsumption": 851675
},
{
- "experimentId": "claude-sonnet-5-opencode--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 85287,
- "measuredAt": "2026-08-28T19:27:58.000Z",
- "meanEstimatedListCostUsd": 0.047190666666666665,
- "meanTokenConsumption": 16207.333333333334,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 73537,
+ "measuredAt": "2026-08-28T22:44:03.395Z",
+ "meanEstimatedListCostUsd": 1.07418,
+ "meanToolInvocationCount": 11,
+ "meanTokenConsumption": 530522
},
{
- "experimentId": "claude-sonnet-5-opencode--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 178438,
- "measuredAt": "2026-08-28T19:27:58.000Z",
- "meanEstimatedListCostUsd": 0.23833933333333335,
- "meanTokenConsumption": 86543,
- "meanToolInvocationCount": 37
+ "meanDurationMs": 78931,
+ "measuredAt": "2026-08-28T22:44:03.405Z",
+ "meanEstimatedListCostUsd": 1.59483,
+ "meanToolInvocationCount": 18,
+ "meanTokenConsumption": 789511
},
{
- "experimentId": "claude-sonnet-5-opencode--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 69785,
- "measuredAt": "2026-08-28T19:20:33.000Z",
- "meanEstimatedListCostUsd": 0.05839,
- "meanTokenConsumption": 16595,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 71745,
+ "measuredAt": "2026-08-28T22:44:03.411Z",
+ "meanEstimatedListCostUsd": 1.021194,
+ "meanToolInvocationCount": 11,
+ "meanTokenConsumption": 499669
},
{
- "experimentId": "claude-sonnet-5-opencode--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 69671,
- "measuredAt": "2026-08-28T19:20:33.000Z",
- "meanEstimatedListCostUsd": 0.08692,
- "meanTokenConsumption": 33568,
- "meanToolInvocationCount": 15.666666666666666
+ "meanDurationMs": 93642,
+ "measuredAt": "2026-08-28T22:44:03.416Z",
+ "meanEstimatedListCostUsd": 1.82048,
+ "meanToolInvocationCount": 17.666666666666668,
+ "meanTokenConsumption": 901060
},
{
- "experimentId": "claude-sonnet-5-opencode--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-006-custom-channel",
"status": "current",
- "passedRuns": 1,
+ "passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 352409,
- "measuredAt": "2026-08-28T19:20:33.000Z",
- "meanEstimatedListCostUsd": 0.017415333333333335,
- "meanTokenConsumption": 5797,
- "meanToolInvocationCount": 6
+ "meanDurationMs": 91912,
+ "measuredAt": "2026-08-28T22:44:03.422Z",
+ "meanEstimatedListCostUsd": 1.172788,
+ "meanToolInvocationCount": 17.333333333333332,
+ "meanTokenConsumption": 578826
},
{
- "experimentId": "claude-sonnet-5-opencode--baseline",
+ "experimentId": "claude-sonnet-5-claude-code--baseline",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 69696,
- "measuredAt": "2026-08-28T19:20:33.000Z",
- "meanEstimatedListCostUsd": 0.06399133333333333,
- "meanTokenConsumption": 21950.333333333332,
- "meanToolInvocationCount": 14
+ "meanDurationMs": 84747,
+ "measuredAt": "2026-08-28T22:44:03.428Z",
+ "meanEstimatedListCostUsd": 1.011918,
+ "meanToolInvocationCount": 13.333333333333334,
+ "meanTokenConsumption": 498895
},
{
- "experimentId": "claude-sonnet-5-opencode--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-001-weather-tool",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 72487,
- "measuredAt": "2026-08-28T19:19:54.000Z",
- "meanEstimatedListCostUsd": 0.06199066666666666,
- "meanTokenConsumption": 20955.333333333332,
- "meanToolInvocationCount": 12.333333333333334
+ "meanDurationMs": 69779,
+ "measuredAt": "2026-08-28T23:16:01.692Z",
+ "meanEstimatedListCostUsd": 1.629858,
+ "meanToolInvocationCount": 12,
+ "meanTokenConsumption": 801201
},
{
- "experimentId": "claude-sonnet-5-opencode--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-002-new-project",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 97671,
- "measuredAt": "2026-08-28T19:25:30.000Z",
- "meanEstimatedListCostUsd": 0.037938000000000006,
- "meanTokenConsumption": 11189,
- "meanToolInvocationCount": 13.666666666666666
+ "meanDurationMs": 56970,
+ "measuredAt": "2026-08-28T23:16:01.703Z",
+ "meanEstimatedListCostUsd": 0.965568,
+ "meanToolInvocationCount": 7.666666666666667,
+ "meanTokenConsumption": 476704
},
{
- "experimentId": "claude-sonnet-5-opencode--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-003-openapi-connection",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 53872,
- "measuredAt": "2026-08-28T19:25:30.000Z",
- "meanEstimatedListCostUsd": 0.06860866666666667,
- "meanTokenConsumption": 27953.666666666668,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 42652,
+ "measuredAt": "2026-08-28T23:16:01.711Z",
+ "meanEstimatedListCostUsd": 0.82867,
+ "meanToolInvocationCount": 8.333333333333334,
+ "meanTokenConsumption": 410267
},
{
- "experimentId": "claude-sonnet-5-opencode--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-004-packaged-skill",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 54347,
- "measuredAt": "2026-08-28T19:19:54.000Z",
- "meanEstimatedListCostUsd": 0.05166133333333334,
- "meanTokenConsumption": 17134.666666666668,
- "meanToolInvocationCount": 10.333333333333334
+ "meanDurationMs": 45265,
+ "measuredAt": "2026-08-28T23:16:01.717Z",
+ "meanEstimatedListCostUsd": 0.67903,
+ "meanToolInvocationCount": 5,
+ "meanTokenConsumption": 332683
},
{
- "experimentId": "claude-sonnet-5-opencode--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-005-conditional-approval",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 57140,
- "measuredAt": "2026-08-28T19:19:54.000Z",
- "meanEstimatedListCostUsd": 0.04402533333333333,
- "meanTokenConsumption": 15630,
- "meanToolInvocationCount": 12
+ "meanDurationMs": 39690,
+ "measuredAt": "2026-08-28T23:16:01.722Z",
+ "meanEstimatedListCostUsd": 0.972688,
+ "meanToolInvocationCount": 7,
+ "meanTokenConsumption": 480972
},
{
- "experimentId": "claude-sonnet-5-opencode--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-006-custom-channel",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 46268,
- "measuredAt": "2026-08-28T19:19:54.000Z",
- "meanEstimatedListCostUsd": 0.026169333333333333,
- "meanTokenConsumption": 8867.333333333334,
- "meanToolInvocationCount": 8
+ "meanDurationMs": 38868,
+ "measuredAt": "2026-08-28T23:16:01.727Z",
+ "meanEstimatedListCostUsd": 1.219158,
+ "meanToolInvocationCount": 7.333333333333333,
+ "meanTokenConsumption": 604483
},
{
- "experimentId": "claude-sonnet-5-opencode--guided",
+ "experimentId": "claude-sonnet-5-claude-code--guided",
"caseId": "author-007-digest-schedule",
"status": "current",
"passedRuns": 3,
"validRuns": 3,
- "meanDurationMs": 27822,
- "measuredAt": "2026-08-28T19:19:54.000Z",
- "meanEstimatedListCostUsd": 0.01684333333333333,
- "meanTokenConsumption": 5760.333333333333,
- "meanToolInvocationCount": 6
+ "meanDurationMs": 68924,
+ "measuredAt": "2026-08-28T23:16:01.731Z",
+ "meanEstimatedListCostUsd": 1.053378,
+ "meanToolInvocationCount": 15,
+ "meanTokenConsumption": 521965
},
{
"experimentId": "glm-5-2-opencode--baseline",
@@ -1314,6 +1330,174 @@
"meanTokenConsumption": 16295.666666666666,
"meanToolInvocationCount": 12.333333333333334
},
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 135534,
+ "measuredAt": "2026-08-29T00:05:25.109Z",
+ "meanEstimatedListCostUsd": 4.933595,
+ "meanToolInvocationCount": 22.666666666666668,
+ "meanTokenConsumption": 947443
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 92582,
+ "measuredAt": "2026-08-29T00:05:25.121Z",
+ "meanEstimatedListCostUsd": 2.50535,
+ "meanToolInvocationCount": 13.333333333333334,
+ "meanTokenConsumption": 488330
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 191566,
+ "measuredAt": "2026-08-29T00:05:25.129Z",
+ "meanEstimatedListCostUsd": 14.008915,
+ "meanToolInvocationCount": 36.333333333333336,
+ "meanTokenConsumption": 2751467
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 125522,
+ "measuredAt": "2026-08-29T00:05:25.138Z",
+ "meanEstimatedListCostUsd": 3.74517,
+ "meanToolInvocationCount": 16,
+ "meanTokenConsumption": 722970
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 135687,
+ "measuredAt": "2026-08-29T00:05:25.146Z",
+ "meanEstimatedListCostUsd": 4.87658,
+ "meanToolInvocationCount": 26.666666666666668,
+ "meanTokenConsumption": 952132
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 202636,
+ "measuredAt": "2026-08-29T00:05:25.152Z",
+ "meanToolInvocationCount": 26.333333333333332,
+ "meanTokenConsumption": 1822465,
+ "meanEstimatedListCostUsd": 9.304045
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--baseline",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 163050,
+ "measuredAt": "2026-08-29T00:05:25.160Z",
+ "meanEstimatedListCostUsd": 5.362835,
+ "meanToolInvocationCount": 26,
+ "meanTokenConsumption": 1044543
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-001-weather-tool",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 106811,
+ "measuredAt": "2026-08-29T00:13:01.331Z",
+ "meanEstimatedListCostUsd": 11.438505,
+ "meanToolInvocationCount": 14.666666666666666,
+ "meanTokenConsumption": 2234333
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-002-new-project",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 91787,
+ "measuredAt": "2026-08-29T00:13:01.344Z",
+ "meanEstimatedListCostUsd": 2.4951,
+ "meanToolInvocationCount": 16,
+ "meanTokenConsumption": 486252
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-003-openapi-connection",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 114352,
+ "measuredAt": "2026-08-29T00:13:01.356Z",
+ "meanEstimatedListCostUsd": 2.90723,
+ "meanToolInvocationCount": 20.666666666666668,
+ "meanTokenConsumption": 568682
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-004-packaged-skill",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 77606,
+ "measuredAt": "2026-08-29T00:13:01.363Z",
+ "meanEstimatedListCostUsd": 2.328845,
+ "meanToolInvocationCount": 10,
+ "meanTokenConsumption": 446685
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-005-conditional-approval",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 104699,
+ "measuredAt": "2026-08-29T00:13:01.370Z",
+ "meanEstimatedListCostUsd": 4.55151,
+ "meanToolInvocationCount": 19,
+ "meanTokenConsumption": 888618
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-006-custom-channel",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 162364,
+ "measuredAt": "2026-08-29T00:13:01.377Z",
+ "meanEstimatedListCostUsd": 6.511585,
+ "meanToolInvocationCount": 16.333333333333332,
+ "meanTokenConsumption": 1266737
+ },
+ {
+ "experimentId": "claude-opus-5-claude-code--guided",
+ "caseId": "author-007-digest-schedule",
+ "status": "current",
+ "passedRuns": 3,
+ "validRuns": 3,
+ "meanDurationMs": 89702,
+ "measuredAt": "2026-08-29T00:13:01.384Z",
+ "meanEstimatedListCostUsd": 3.44083,
+ "meanToolInvocationCount": 16.333333333333332,
+ "meanTokenConsumption": 674858
+ },
{
"experimentId": "gemini-3-1-pro-preview-opencode--baseline",
"caseId": "author-001-weather-tool",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 4f0eb8462..f89d9f081 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -139,20 +139,23 @@ importers:
apps/benchmarks:
dependencies:
'@ai-sdk/harness':
- specifier: 1.0.72
+ specifier: 1.0.87
+ '@ai-sdk/harness-claude-code':
+ specifier: 1.0.90
'@ai-sdk/harness-opencode':
- specifier: 1.0.73
+ specifier: 1.0.88
'@ai-sdk/sandbox-vercel':
- specifier: 1.0.72
+ specifier: 1.0.87
'@vercel/agent-eval':
specifier: 1.5.0
version: 1.5.0([email protected])
'@vercel/sandbox':
- specifier: 'catalog:'
- version: 2.8.0
+ specifier: 3.2.0
+ version: 3.2.0
jiti:
specifier: 2.7.0
version: 2.7.0
@@ -253,10 +256,10 @@ importers:
version: 0.33.0(@vercel/[email protected])(eve@packages+eve)
'@agentphone/chat-sdk-adapter':
specifier: 0.1.0
'@beeper/chat-adapter-matrix':
specifier: 0.2.0
'@blitzreels/eve':
specifier: ^0.2.1
version: 0.2.1(eve@packages+eve)
@@ -265,49 +268,49 @@ importers:
version: 0.1.0(@cfworker/[email protected])(eve@packages+eve)([email protected])
'@chat-adapter/gchat':
specifier: 4.34.0
'@chat-adapter/messenger':
specifier: 4.34.0
'@chat-adapter/state-memory':
specifier: 4.34.0
'@chat-adapter/whatsapp':
specifier: 4.34.0
'@chat-adapter/x':
specifier: 4.34.0
'@eve/self-modification':
specifier: workspace:*
version: link:../../packages/eve-self-modification
'@getdial/chat-sdk-adapter':
specifier: 0.2.0
'@github-tools/eve-extension':
specifier: ^0.1.0
'@jetty/eve':
specifier: ^0.3.0
version: 0.3.0(eve@packages+eve)
'@kapso/chat-adapter':
specifier: 0.1.1
'@kybernesis/arcana':
specifier: 0.2.0
version: 0.2.0(eve@packages+eve)
'@larksuite/vercel-chat-adapter':
specifier: 0.1.2
'@liveblocks/chat-sdk-adapter':
specifier: 3.23.0
'@novu/chat-sdk-adapter':
specifier: 0.1.0
'@onkernel/eve-extension':
specifier: ^0.1.3
'@opentelemetry/api':
specifier: 1.9.1
version: 1.9.1
@@ -322,7 +325,7 @@ importers:
'@resend/chat-sdk-adapter':
specifier: 0.2.2
'@tailwindcss/postcss':
specifier: 4.3.0
version: 4.3.0
@@ -352,10 +355,10 @@ importers:
version: 0.2.1(eve@packages+eve)
'@veltdev/chat-sdk-adapter':
specifier: 0.2.4
'@vercel/connect':
specifier: 'catalog:'
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
@@ -367,22 +370,22 @@ importers:
version: 0.1.71
'@zernio/chat-sdk-adapter':
specifier: 0.5.0
braintrust:
specifier: 3.24.0
chat:
specifier: 4.34.0
chat-adapter-sendblue:
specifier: 0.2.0
eve:
specifier: workspace:*
version: link:../../packages/eve
eve-channel-blooio:
specifier: 0.2.0
- version: 0.2.0([email protected]([email protected]))(eve@packages+eve)
+ version: 0.2.0([email protected]([email protected]))(eve@packages+eve)
postcss:
specifier: 8.5.15
version: 8.5.15
@@ -690,7 +693,7 @@ importers:
version: 25.9.1
historical-eve-0-30-8:
specifier: npm:[email protected]
typescript:
specifier: 'catalog:'
version: 7.0.2
@@ -1619,8 +1622,8 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-SXUM8jzzuTUJRq+EOgPd5to6DSx0EKslVn+IVZHbUEX6k/3vCPNrvjckbK26HnNxHU/STxm+zTSJteqrO+7Z0w==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-iayK1z11I0b3sgbP3j6eKQypfpwO5Fbtaq1Mkwv7YxwGQG1YTNr5QAjyYsSfqyf/58WUOLWz2cTBpk+ExZVgpw==}
engines: {node: '>=22'}
peerDependencies:
zod: ^3.25.76 || ^4.1.8
@@ -1649,14 +1652,20 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-OewZBK5NuufxY9Ve9VahtQ2ma3oaiZXO89iXB5rmzyyF4kMCx5nDNirAwY/HaR8rcwhUE4gGGuoBjNUNPHkaYA==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-SL77y5l3596n6cxuLN4OB19iTkvwg6zJq2KxIV9MAPwAMYEEXHz7YGRXxrAd7GOixIVqcsXEw1HOLN1zcG1L3A==}
+ engines: {node: '>=22'}
+ peerDependencies:
+ zod: ^3.25.76 || ^4.1.8
+
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-bA22/rBaTbsO/o/x91P78RvLpvgwnCoW6Ie7ATsC7Z6BxGYZpFrS6BGWD1DQL2pN3jy61KWnDLDwy0egg7ypZg==}
engines: {node: '>=22'}
peerDependencies:
zod: ^3.25.76 || ^4.1.8
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-p11kq04JlKCy6VCjzhyeGZJhQbdYkGYdkCOLspFVoBYoW7hp2Co6nob8o2l1uRAsGTFqGdWZ6ZFaKov1K+BjKA==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-OY+M1B58lLv+fRHB0dh/FJ6lCzmg9HQmTe1z+XS1WTPF8PnTMEr2E/6vNTUoORN7iZMkTyVkszA4DOhod1eFQw==}
engines: {node: '>=22'}
peerDependencies:
ws: ^8.21.0
@@ -1729,8 +1738,8 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-EzAn4pdgG5g0xXtH6lE2zyNmfjDQIDjATkfqzuidEI35g++hh4+07vnjzkT/RmGmIClPZiRj/Q2GMPV2V7mkHw==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-9TyxUXolql77ntHIeygLqt7PO1O0HZPB73cJhLG2OsPecSBRIZhNXLxT1T7rlL6Zpm1eDoLMFrMV99kAJ28/Sg==}
engines: {node: '>=22'}
peerDependencies:
zod: ^3.25.76 || ^4.1.8
@@ -1751,14 +1760,18 @@ packages:
resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==}
engines: {node: '>=22'}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-aWO7iwhFUGf347tCwNGggggfmZigaSu7TF739IZSrWWABUp7zkb4Cr3fMqvBe5EIS7ABJJu3Cadn0g/zs1G0QQ==}
+ engines: {node: '>=22'}
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-El0jUZ/B7mvBHAD5rfSDqOAhWxutVTq7BCNhfGuwfDPT9SO0TMHybh2bMkieJQI7YOfl+qNBoWrRAOHHaFb99Q==}
engines: {node: '>=18'}
peerDependencies:
react: ^18 || ~19.0.1 || ~19.1.2 || ^19.2.1
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-AolOfKEZ5VhsEXRVwDGroabY5Fy2fCNxEFFYpdjqd4J/1IxgpBSgoYNWOS0T1Xg/0J4WCccAM7PQwpmJgwLtVw==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-rW8mtwvH5lcoXWPn8hEdh2Kns/2qAQokS9xG/UeqZOgp79ifKJQp8dvWpUSzRNZKgPtqsXNmVFac6J/UnlmO+Q==}
engines: {node: '>=22'}
'@ai-sdk/[email protected]':
@@ -8495,8 +8508,8 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
- resolution: {integrity: sha512-wBUyoCYF3GVr+62nelBgR8YbpTSsMZrzFyOOjiwijylNSM2TFCW35C+Pml2vc59/WLMpyhS/LWZ55M+B9DAcSg==}
+ resolution: {integrity: sha512-zZkSUq6MgmXjwIfML2wkPFeQ5Azk8kGEWkciAtE0NgnFy3slj4ljsw5LvigsCbYXxTSVr+6iCHBTPZafpx1P0w==}
engines: {node: '>=22'}
peerDependencies:
zod: ^3.25.76 || ^4.1.8
@@ -15921,10 +15934,10 @@ snapshots:
dependencies:
zod: 4.4.3
dependencies:
transitivePeerDependencies:
- ai
- supports-color
@@ -16012,10 +16025,10 @@ snapshots:
'@vercel/oidc': 3.2.0
zod: 4.4.3
dependencies:
- '@ai-sdk/provider': 4.0.7
- '@ai-sdk/provider-utils': 5.0.27([email protected])
+ '@ai-sdk/provider': 4.0.8
+ '@ai-sdk/provider-utils': 5.0.30([email protected])
'@vercel/oidc': 3.2.0
zod: 4.4.3
@@ -16052,25 +16065,44 @@ snapshots:
zod: 4.4.3
optional: true
dependencies:
- '@ai-sdk/provider-utils': 5.0.27([email protected])
+ '@ai-sdk/provider-utils': 5.0.30([email protected])
+ ws: 8.21.3([email protected])
+ zod: 4.4.3
+ transitivePeerDependencies:
+ - bufferutil
+ - utf-8-validate
+
+ dependencies:
+ '@ai-sdk/provider-utils': 5.0.30([email protected])
ws: 8.21.0([email protected])
zod: 4.4.3
transitivePeerDependencies:
- bufferutil
- utf-8-validate
dependencies:
- '@ai-sdk/provider': 4.0.7
- '@ai-sdk/provider-utils': 5.0.27([email protected])
- ai: 7.0.66([email protected])
+ '@ai-sdk/provider': 4.0.8
+ '@ai-sdk/provider-utils': 5.0.30([email protected])
+ ai: 7.0.79([email protected])
zod: 4.4.3
optionalDependencies:
ws: 8.21.0([email protected])
+ dependencies:
+ '@ai-sdk/provider': 4.0.8
+ '@ai-sdk/provider-utils': 5.0.30([email protected])
+ ai: 7.0.79([email protected])
+ zod: 4.4.3
+ optionalDependencies:
+ ws: 8.21.3([email protected])
+
dependencies:
'@ai-sdk/provider': 4.0.7
@@ -16157,9 +16189,9 @@ snapshots:
undici: 7.29.0
zod: 4.4.3
dependencies:
- '@ai-sdk/provider': 4.0.7
+ '@ai-sdk/provider': 4.0.8
'@standard-schema/spec': 1.1.0
'@workflow/serde': 4.1.0
eventsource-parser: 3.1.0
@@ -16182,6 +16214,10 @@ snapshots:
dependencies:
json-schema: 0.4.0
+ '@ai-sdk/[email protected]':
+ dependencies:
+ json-schema: 0.4.0
+
dependencies:
'@ai-sdk/provider-utils': 4.0.27([email protected])
@@ -16192,11 +16228,11 @@ snapshots:
transitivePeerDependencies:
- zod
dependencies:
- '@ai-sdk/provider-utils': 5.0.27([email protected])
- '@vercel/sandbox': 2.8.0
+ '@ai-sdk/provider-utils': 5.0.30([email protected])
+ '@vercel/sandbox': 3.2.0
transitivePeerDependencies:
- bare-abort-controller
- react-native-b4a
@@ -16532,11 +16568,11 @@ snapshots:
'@balena/[email protected]': {}
dependencies:
- '@chat-adapter/state-redis': 4.35.0(@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])
+ '@chat-adapter/state-redis': 4.35.0(@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])
marked: 15.0.12
matrix-js-sdk: 41.9.0
node-html-parser: 7.1.0
@@ -16786,29 +16822,29 @@ snapshots:
human-id: 4.1.3
prettier: 2.8.8
dependencies:
'@googleapis/chat': 44.7.0([email protected])
'@googleapis/workspaceevents': 9.2.0([email protected])
transitivePeerDependencies:
- ai
- supports-color
- zod
dependencies:
transitivePeerDependencies:
- ai
- supports-color
- zod
dependencies:
transitivePeerDependencies:
- ai
- supports-color
@@ -16822,9 +16858,9 @@ snapshots:
- supports-color
- zod
dependencies:
transitivePeerDependencies:
- ai
- supports-color
@@ -16859,12 +16895,12 @@ snapshots:
- zod
optional: true
dependencies:
transitivePeerDependencies:
- ai
- bufferutil
@@ -16882,17 +16918,17 @@ snapshots:
- supports-color
- zod
dependencies:
transitivePeerDependencies:
- ai
- supports-color
- zod
dependencies:
redis: 5.12.1(@opentelemetry/[email protected])
transitivePeerDependencies:
- '@node-rs/xxhash'
@@ -16911,19 +16947,19 @@ snapshots:
- supports-color
- zod
dependencies:
transitivePeerDependencies:
- ai
- supports-color
- zod
dependencies:
transitivePeerDependencies:
- ai
- supports-color
@@ -17414,11 +17450,11 @@ snapshots:
dependencies:
tslib: 2.8.1
dependencies:
transitivePeerDependencies:
- ai
- encoding
@@ -17435,26 +17471,26 @@ snapshots:
- react-native
- supports-color
dependencies:
eve: link:packages/eve
zod: 4.4.3
optionalDependencies:
transitivePeerDependencies:
- '@ai-sdk/workflow'
- '@workflow/ai'
- ai
- workflow
dependencies:
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
octokit: 5.0.5
zod: 4.4.3
optionalDependencies:
eve: link:packages/eve
@@ -17833,11 +17869,11 @@ snapshots:
'@js-sdsl/[email protected]': {}
dependencies:
'@kapso/whatsapp-cloud-api': 0.2.3
transitivePeerDependencies:
- ai
- supports-color
@@ -17909,11 +17945,11 @@ snapshots:
'@langchain/[email protected]': {}
dependencies:
'@larksuiteoapi/node-sdk': 1.63.1([email protected])([email protected]([email protected]))
transitivePeerDependencies:
- ai
- bufferutil
@@ -17946,11 +17982,11 @@ snapshots:
dependencies:
standardwebhooks: 1.0.0
dependencies:
'@liveblocks/core': 3.23.0(@types/[email protected])
'@liveblocks/node': 3.23.0(@types/[email protected])
transitivePeerDependencies:
- '@types/json-schema'
- encoding
@@ -18308,10 +18344,10 @@ snapshots:
'@nolyfill/[email protected]': {}
dependencies:
optionalDependencies:
react: 19.2.6
transitivePeerDependencies:
@@ -18942,9 +18978,9 @@ snapshots:
'@octokit/request-error': 7.1.0
'@octokit/webhooks-methods': 6.0.0
dependencies:
eve: link:packages/eve
zod: 4.4.3
transitivePeerDependencies:
@@ -20963,10 +20999,10 @@ snapshots:
'@repeaterjs/[email protected]': {}
dependencies:
hast-util-to-html: 9.0.5
mdast-util-to-hast: 13.2.1
@@ -22270,10 +22306,10 @@ snapshots:
dependencies:
eve: link:packages/eve
dependencies:
hast-util-from-html: 2.0.3
hast-util-to-html: 9.0.5
hast-util-to-mdast: 10.1.2
@@ -22414,14 +22450,14 @@ snapshots:
dependencies:
execa: 5.1.1
dependencies:
'@vercel/oidc': 3.8.1
optionalDependencies:
'@ai-sdk/mcp': 2.0.29([email protected])
'@auth/core': 0.41.2
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
eve: link:packages/eve
@@ -22434,14 +22470,14 @@ snapshots:
ai: 7.0.58([email protected])
eve: link:packages/eve
dependencies:
'@vercel/oidc': 3.8.5
optionalDependencies:
'@ai-sdk/mcp': 2.0.29([email protected])
'@auth/core': 0.41.2
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
eve: link:packages/eve
@@ -23358,10 +23394,10 @@ snapshots:
ulid: 3.0.2
zod: 4.3.6
dependencies:
transitivePeerDependencies:
- ai
- supports-color
@@ -23442,11 +23478,11 @@ snapshots:
'@ai-sdk/provider-utils': 5.0.25([email protected])
zod: 4.4.3
dependencies:
- '@ai-sdk/gateway': 4.0.52([email protected])
- '@ai-sdk/provider': 4.0.7
- '@ai-sdk/provider-utils': 5.0.27([email protected])
+ '@ai-sdk/gateway': 4.0.64([email protected])
+ '@ai-sdk/provider': 4.0.8
+ '@ai-sdk/provider-utils': 5.0.30([email protected])
zod: 4.4.3
@@ -24041,12 +24077,12 @@ snapshots:
dependencies:
sendblue: 3.10.1
dependencies:
'@workflow/serde': 4.1.0-beta.2
mdast-util-to-string: 4.0.0
@@ -24056,7 +24092,7 @@ snapshots:
remend: 1.3.0
unified: 11.0.5
optionalDependencies:
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
zod: 4.4.3
transitivePeerDependencies:
- supports-color
@@ -24076,7 +24112,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
dependencies:
'@workflow/serde': 4.1.0-beta.2
mdast-util-to-string: 4.0.0
@@ -24086,12 +24122,12 @@ snapshots:
remend: 1.3.0
unified: 11.0.5
optionalDependencies:
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
zod: 4.4.3
transitivePeerDependencies:
- supports-color
dependencies:
'@workflow/serde': 4.1.0-beta.2
mdast-util-to-string: 4.0.0
@@ -24101,7 +24137,7 @@ snapshots:
remend: 1.3.0
unified: 11.0.5
optionalDependencies:
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
zod: 4.4.3
transitivePeerDependencies:
- supports-color
@@ -25577,9 +25613,9 @@ snapshots:
dependencies:
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
eve: link:packages/eve
@@ -25632,9 +25668,9 @@ snapshots:
- xml2js
- zephyr-agent
dependencies:
- ai: 7.0.66([email protected])
+ ai: 7.0.79([email protected])
undici: 8.9.0
optionalDependencies: