The Open Framework for Building Agents
docs(benchmarks) anthropic to use claude harness introduce opus 5 (#2731)
Colton Padden83d40081dd7c
diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjsindex e6d68c63b..dc550aafa 100644--- a/apps/benchmarks/lib/benchmark-config.test.mjs+++ b/apps/benchmarks/lib/benchmark-config.test.mjs@@ -19,6 +19,7 @@ test("publishes only compatibility-validated models", () => {"gpt-5-6-terra","claude-sonnet-5","glm-5-2",+ "claude-opus-5","gemini-3-1-pro-preview",],);@@ -34,6 +35,10 @@ test("publishes only compatibility-validated models", () => {assert.equal(findPublishedBenchmarkModel("grok-4-6").model, "xai/grok-4.6");assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").model, "anthropic/claude-sonnet-5");assert.equal(findPublishedBenchmarkModel("glm-5-2").model, "zai/glm-5.2");+ assert.equal(findPublishedBenchmarkModel("claude-fable-5").harness, "Claude Code");+ assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code");+ assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code");+ assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode");});test("allows candidate probes and rejects unknown models", () => {@@ -41,6 +46,6 @@ test("allows candidate probes and rejects unknown models", () => {assert.equal(findBenchmarkModel("gpt-5-6-terra").support, "supported");assert.equal(findBenchmarkModel("grok-4-6").support, "supported");assert.equal(findBenchmarkModel("claude-sonnet-4-6").support, "superseded");- assert.equal(findBenchmarkModel("claude-opus-5").support, "candidate");+ assert.equal(findBenchmarkModel("claude-opus-5").support, "supported");assert.throws(() => findBenchmarkModel("unknown"), /Unknown model/u);});diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.tsindex 3abb3cfbc..eb2185b75 100644--- a/apps/benchmarks/lib/benchmark-config.ts+++ b/apps/benchmarks/lib/benchmark-config.ts@@ -8,7 +8,7 @@ export interface AuthoringBenchmarkModel {readonly id: string;readonly model: string;readonly displayName: string;- readonly harness: "OpenCode";+ readonly harness: "OpenCode" | "Claude Code";readonly support: AuthoringBenchmarkSupport;}@@ -31,7 +31,7 @@ export const benchmarkModels = [id: "claude-fable-5",model: "anthropic/claude-fable-5",displayName: "Claude Fable 5",- harness: "OpenCode",+ harness: "Claude Code",support: "supported",},{@@ -59,7 +59,7 @@ export const benchmarkModels = [id: "claude-sonnet-5",model: "anthropic/claude-sonnet-5",displayName: "Claude Sonnet 5",- harness: "OpenCode",+ harness: "Claude Code",support: "supported",},{@@ -73,8 +73,8 @@ export const benchmarkModels = [id: "claude-opus-5",model: "anthropic/claude-opus-5",displayName: "Claude Opus 5",- harness: "OpenCode",- support: "candidate",+ harness: "Claude Code",+ support: "supported",},{id: "gemini-3-1-pro-preview",@@ -103,10 +103,14 @@ export const publishedBenchmark = {} as const;export function publishedExperimentId(- benchmark: Pick<AuthoringBenchmarkModel, "id">,+ benchmark: Pick<AuthoringBenchmarkModel, "id" | "harness">,treatment: AuthoringTreatment,): string {- return `${benchmark.id}-opencode--${treatment}`;+ return `${benchmark.id}-${harnessId(benchmark.harness)}--${treatment}`;+}++export function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {+ return harness === "Claude Code" ? "claude-code" : "opencode";}export function parseAuthoringTreatment(value: string): AuthoringTreatment {diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.tsindex f18c9a4bc..71e6f8bdb 100644--- a/apps/benchmarks/lib/experiment.ts+++ b/apps/benchmarks/lib/experiment.ts@@ -16,6 +16,7 @@ export function authoringExperiment(options: {readonly verbose?: boolean;}): ExperimentConfig {const agent = createAuthoringAgent({+ harness: options.benchmark.harness,model: options.benchmark.model,archive: options.archive,dependencyArchive: options.dependencyArchive,diff --git a/apps/benchmarks/lib/harness-agent.test.mjs b/apps/benchmarks/lib/harness-agent.test.mjsnew file mode 100644index 000000000..c49a19937--- /dev/null+++ b/apps/benchmarks/lib/harness-agent.test.mjs@@ -0,0 +1,56 @@+import assert from "node:assert/strict";+import { test } from "node:test";++import { createJiti } from "jiti";++const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });+const { addUsage } = await jiti.import(new URL("./harness-agent.ts", import.meta.url).pathname);++test("normalizes nested Claude Code token usage", () => {+ const usage = {+ inputTokens: 0,+ outputTokens: 0,+ reasoningTokens: 0,+ cachedInputTokens: 0,+ cacheWriteTokens: 0,+ };++ addUsage(usage, {+ inputTokens: { total: 32_177, noCache: 1, cacheRead: 32_176, cacheWrite: 10_944 },+ outputTokens: { total: 327, text: 327 },+ });++ assert.deepEqual(usage, {+ inputTokens: 1,+ outputTokens: 327,+ reasoningTokens: 0,+ cachedInputTokens: 32_176,+ cacheWriteTokens: 10_944,+ });+});++test("retains flat OpenCode token usage", () => {+ const usage = {+ inputTokens: 0,+ outputTokens: 0,+ reasoningTokens: 0,+ cachedInputTokens: 0,+ cacheWriteTokens: 0,+ };++ addUsage(usage, {+ inputTokens: 100,+ outputTokens: 20,+ reasoningTokens: 5,+ cachedInputTokens: 50,+ cacheWriteTokens: 10,+ });++ assert.deepEqual(usage, {+ inputTokens: 100,+ outputTokens: 20,+ reasoningTokens: 5,+ cachedInputTokens: 50,+ cacheWriteTokens: 10,+ });+});diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.tsindex c2feb69a8..eacbb47f0 100644--- a/apps/benchmarks/lib/harness-agent.ts+++ b/apps/benchmarks/lib/harness-agent.ts@@ -1,7 +1,8 @@import { readFileSync } from "node:fs";-import type { HarnessV1NetworkSandboxSession } from "@ai-sdk/harness";+import type { HarnessV1, HarnessV1NetworkSandboxSession } from "@ai-sdk/harness";import type { HarnessAgentSession } from "@ai-sdk/harness/agent";import { HarnessAgent } from "@ai-sdk/harness/agent";+import { createClaudeCode } from "@ai-sdk/harness-claude-code";import { createOpenCode } from "@ai-sdk/harness-opencode";import type { Agent, AgentRunResult } from "@vercel/agent-eval";import { z } from "zod";@@ -12,6 +13,7 @@ import type {AuthoringSetupContext,AuthoringTurn,} from "./authoring-case.js";+import type { AuthoringBenchmarkModel } from "./benchmark-config.js";import { createDependencyCachedSandbox } from "./dependency-sandbox.js";import { loadAuthoringCase } from "./load-authoring-case.js";import {@@ -28,15 +30,15 @@ import { BenchmarkTimings } from "./timing.js";const HARNESS_BRIDGE_PORT = 4172;const POST_RUN_GRADER_DIRECTORY = ".eve-grader";-const BOOTSTRAP_VERSION = "v8";+const BOOTSTRAP_VERSION = "v9";// A turn that stops producing output should end the turn, not the eval: the// remaining turns still run and the graders still see what the agent did.-const TURN_TIMEOUT_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_TIMEOUT ?? 480);+const TURN_TIMEOUT_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_TIMEOUT ?? 300);// A turn that has produced no chunk for this long is waiting on the runtime, not// working. The bound is one slow tool call: the bridge reports a call and its// result together once the call returns, so an install that takes minutes looks// from here like silence rather than work in flight.-const TURN_STALL_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_STALL ?? 240);+const TURN_STALL_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_STALL ?? 120);// The runtime frequently never closes a turn whose last act was the model// talking, so silence after a closing message has to end the turn on its own.// Long enough that a pause before the next tool call is not mistaken for one.@@ -49,10 +51,7 @@ const TURN_SETTLE_MILLIS = 15_000;// the turn, so the harness answers it immediately and points the agent at the// channel this benchmark can answer on: its reply, which the case's next `send`// responds to.-type AuthoringHarnessAgent = HarnessAgent<- ReturnType<typeof createOpenCode>,- typeof INTERACTIVE_QUESTION_TOOL->;+type AuthoringHarnessAgent = HarnessAgent<HarnessV1, typeof INTERACTIVE_QUESTION_TOOL>;const INTERACTIVE_QUESTION_TOOL = {question: {@@ -64,6 +63,7 @@ const INTERACTIVE_QUESTION_TOOL = {};export function createAuthoringAgent(subject: {+ readonly harness: AuthoringBenchmarkModel["harness"];readonly model: string;readonly archive: Uint8Array;readonly dependencyArchive: Uint8Array;@@ -71,15 +71,15 @@ export function createAuthoringAgent(subject: {readonly dependencyDigest: string;}): Agent {return {- name: "opencode",- displayName: "eve authoring harness",+ name: harnessId(subject.harness),+ displayName: `${subject.harness} eve authoring harness`,getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY",getDefaultModel: () => subject.model,definition: {- name: "opencode",- displayName: "eve authoring harness",+ name: harnessId(subject.harness),+ displayName: `${subject.harness} eve authoring harness`,defaultModel: subject.model,- o11yAgentName: "opencode",+ o11yAgentName: harnessId(subject.harness),runnerPath: "",getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY",install: () => [],@@ -126,11 +126,7 @@ export function createAuthoringAgent(subject: {const agent = new HarnessAgent({id: "eve-authoring-eval",- harness: createOpenCode({- auth: "ai-gateway",- model: openCodeModel(options.model ?? subject.model),- port: HARNESS_BRIDGE_PORT,- }),+ harness: createHarness(subject.harness, options.model ?? subject.model),sandbox,tools: INTERACTIVE_QUESTION_TOOL,sandboxConfig: {@@ -159,6 +155,12 @@ export function createAuthoringAgent(subject: {activeSandbox = current as HarnessV1NetworkSandboxSession;workspace = sessionWorkDir;const context = setupContext(activeSandbox, workspace);+ await ensureWorkspace(+ context,+ authoringCase.startingPoint.workspace,+ subject.archive,+ timings,+ );await timings.measure("session.setup", async () => {for (const setup of setups) await setup.onSession?.(context);if (options.agentOptions?.agentsMd !== true) await installBaselineEveWrapper(context);@@ -168,15 +170,6 @@ export function createAuthoringAgent(subject: {}},},- // OpenCode subagents run outside the observed harness session: their tool- // calls and token usage are not captured, and the pinned adapter can hang- // waiting for the delegated task to return. Keep every model in one- // session using direct filesystem and shell tools so runs stay observable- // and comparable. Ending promptly after implementation also prevents a- // completed change from becoming an infrastructure timeout while the- // model continues unrequested exploration or validation.- instructions:- "Work directly in the current session. Do not delegate to subagents; use the available filesystem and shell tools yourself. Once the requested implementation is complete, stop promptly without additional exploration or validation unless the user requested it.",permissionMode: "allow-all",});@@ -192,9 +185,9 @@ export function createAuthoringAgent(subject: {session,transcript,send: async (prompt) => {+ const turn = transcript.filter((entry) => entry.role === "user").length + 1;transcript.push({ role: "user", content: prompt });if (verbose) console.log(`[user] ${prompt}`);- const turn = transcript.filter((entry) => entry.role === "user").length;const result = await timings.measure(`agent.turn.${turn}`, () =>runTurn({agent,@@ -446,6 +439,7 @@ async function runTurn(input: {turnLooksDone = !stepCalledTool;stepCalledTool = false;} else if (part.type === "finish") {+ replaceUsage(usage, (part as { totalUsage?: unknown }).totalUsage);turnLooksDone = true;}}@@ -502,13 +496,60 @@ async function withDeadline<T>(promise: Promise<T>, millis: number): Promise<T |}}-function addUsage(total: Record<string, number>, value: unknown): void {+type MutableAuthoringTokenUsage = {+ -readonly [Key in keyof AuthoringTokenUsage]: AuthoringTokenUsage[Key];+};++export function addUsage(total: MutableAuthoringTokenUsage, value: unknown): void {if (typeof value !== "object" || value === null) return;- for (const [field, amount] of Object.entries(value as Record<string, unknown>)) {- if (typeof amount === "number" && Number.isFinite(amount) && total[field] !== undefined) {- total[field] += amount;- }+ const usage = value as Record<string, unknown>;+ const input = usage.inputTokens;+ const output = usage.outputTokens;+ if (typeof input === "object" && input !== null) {+ const tokens = input as Record<string, unknown>;+ total.inputTokens += usageNumber(tokens.noCache ?? tokens.total);+ total.cachedInputTokens += usageNumber(tokens.cacheRead);+ total.cacheWriteTokens += usageNumber(tokens.cacheWrite);+ } else {+ total.inputTokens += usageNumber(input);+ }+ if (typeof output === "object" && output !== null) {+ total.outputTokens += usageNumber((output as Record<string, unknown>).total);+ } else {+ total.outputTokens += usageNumber(output);+ }+ total.reasoningTokens += usageNumber(usage.reasoningTokens);+ total.cachedInputTokens += usageNumber(usage.cachedInputTokens);+ total.cacheWriteTokens += usageNumber(usage.cacheWriteTokens);+}++function replaceUsage(total: MutableAuthoringTokenUsage, value: unknown): void {+ if (typeof value !== "object" || value === null) return;+ total.inputTokens = 0;+ total.outputTokens = 0;+ total.reasoningTokens = 0;+ total.cachedInputTokens = 0;+ total.cacheWriteTokens = 0;+ addUsage(total, value);+}++function usageNumber(value: unknown): number {+ return typeof value === "number" && Number.isFinite(value) ? value : 0;+}++function createHarness(harness: AuthoringBenchmarkModel["harness"], model: string): HarnessV1 {+ if (harness === "Claude Code") {+ return createClaudeCode({ auth: { gateway: {} }, model });}+ return createOpenCode({+ auth: "ai-gateway",+ model: openCodeModel(model),+ port: HARNESS_BRIDGE_PORT,+ });+}++function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {+ return harness === "Claude Code" ? "claude-code" : "opencode";}function openCodeModel(model: string): string {@@ -555,15 +596,8 @@ async function bootstrapSubject(if (workspaceKind === "scaffolded") {workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`);}- // Grader files must not change what subject commands observe in the project directory.- workspaceCommands.push(- `rm -rf ${AGENT_EVAL_DIRECTORY} && mkdir -p ${AGENT_EVAL_DIRECTORY}`,- `printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,- "command -v vitest >/dev/null",- );- if (workspaceKind === "scaffolded") {- workspaceCommands.push(`test -f ${workspace}/package.json`);- }+ // Grader files are created after the agent finishes so an empty starting point remains empty.+ workspaceCommands.push("command -v vitest >/dev/null");await timings.measure("subject.workspace-bootstrap", () =>run(sandbox, workspaceCommands.join(" && ")),);@@ -592,6 +626,44 @@ function setupContext(// Created only after the agent's turns finish: an `empty` starting point has to// look empty to `eve init .`, which refuses to scaffold into a directory that// already holds entries it does not recognize.+async function ensureWorkspace(+ context: AuthoringSetupContext,+ workspaceKind: "scaffolded" | "empty",+ archive: Uint8Array,+ timings: BenchmarkTimings,+): Promise<void> {+ if (workspaceKind === "empty") {+ await verifyWorkspace(context, workspaceKind);+ return;+ }+ if (await hasPreparedWorkspace(context)) return;++ await timings.measure("session.workspace-recovery", () =>+ bootstrapSubject(context.sandbox, context.workspace, workspaceKind, archive, timings),+ );+ await verifyWorkspace(context, workspaceKind);+}++async function hasPreparedWorkspace(context: AuthoringSetupContext): Promise<boolean> {+ const result = await resultOf(+ context.sandbox,+ `test -f package.json && test -f agent/instructions.md && test -f AGENTS.md && test -f ${EVE_PACKAGE_PATH}`,+ context.workspace,+ );+ return result.exitCode === 0;+}++async function verifyWorkspace(+ context: AuthoringSetupContext,+ workspaceKind: "scaffolded" | "empty",+): Promise<void> {+ const command =+ workspaceKind === "empty"+ ? "test ! -e package.json && test ! -e AGENTS.md && test ! -e agent && test ! -e .eve-grader"+ : `for path in package.json agent/instructions.md AGENTS.md ${EVE_PACKAGE_PATH}; do test -f "$path" || { echo "Missing prepared workspace file: $path" >&2; exit 1; }; done`;+ await context.run(command);+}+async function prepareGraderDirectory(context: AuthoringSetupContext): Promise<void> {await context.run(`mkdir -p ${AGENT_EVAL_DIRECTORY} && printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.jsonindex 57b72d598..348170ae9 100644--- a/apps/benchmarks/package.json+++ b/apps/benchmarks/package.json@@ -12,11 +12,12 @@"benchmark:timings": "node scripts/timings.mjs"},"dependencies": {- "@ai-sdk/harness": "1.0.72",- "@ai-sdk/harness-opencode": "1.0.73",- "@ai-sdk/sandbox-vercel": "1.0.72",+ "@ai-sdk/harness": "1.0.87",+ "@ai-sdk/harness-claude-code": "1.0.90",+ "@ai-sdk/harness-opencode": "1.0.88",+ "@ai-sdk/sandbox-vercel": "1.0.87","@vercel/agent-eval": "1.5.0",- "@vercel/sandbox": "catalog:",+ "@vercel/sandbox": "3.2.0","jiti": "2.7.0","zod": "catalog:"},diff --git a/apps/benchmarks/scripts/cost.mjs b/apps/benchmarks/scripts/cost.mjsindex 66119b0fd..3a4e840c3 100644--- a/apps/benchmarks/scripts/cost.mjs+++ b/apps/benchmarks/scripts/cost.mjs@@ -49,6 +49,14 @@ export const modelPricing = {cacheRead: 0.2,cacheWrite: 2.5,},+ "anthropic/claude-opus-5": {+ effectiveDate: "2026-08-28",+ source: "https://models.dev/anthropic/claude-opus-5",+ input: 5,+ output: 25,+ cacheRead: 0.5,+ cacheWrite: 6.25,+ },"anthropic/claude-sonnet-5": {effectiveDate: "2026-08-17",source: "https://docs.anthropic.com/en/docs/about-claude/pricing",@@ -97,7 +105,7 @@ export function extractRunUsage(raw) {// Ignore malformed transcript lines.}}- return found ? usage : null;+ return found && tokenConsumption(usage) > 0 ? usage : null;}export function tokenConsumption(usage) {diff --git a/apps/benchmarks/scripts/export-results.mjs b/apps/benchmarks/scripts/export-results.mjsindex 0d1c71e5b..bf1e4f505 100644--- a/apps/benchmarks/scripts/export-results.mjs+++ b/apps/benchmarks/scripts/export-results.mjs@@ -21,6 +21,7 @@ import {publishedBenchmark,publishedBenchmarkModels,publishedExperimentId,+ harnessId,} from "../lib/benchmark-config.ts";const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");@@ -98,7 +99,7 @@ const output = {experiments: benchmarks.flatMap((benchmark) =>authoringTreatments.map((treatment) => ({id: publishedExperimentId(benchmark, treatment),- groupId: `${benchmark.id}-opencode`,+ groupId: `${benchmark.id}-${harnessId(benchmark.harness)}`,model: benchmark.model,modelDisplayName: benchmark.displayName,harness: benchmark.harness,@@ -131,12 +132,26 @@ function readPreviousResults() {const supersededExperiments = (previous.experiments ?? []).filter((experiment) =>supersededExperimentIds.has(experiment.id),);+ const archivedExperimentIds = new Set(supersededExperiments.map((experiment) => experiment.id));const supersededResults = (previous.results ?? []).filter((result) =>- supersededExperimentIds.has(result.experimentId),- );- const retained = (previous.previouslyMeasured ?? []).filter(- (measurement) => Array.isArray(measurement.experiments) && Array.isArray(measurement.results),+ archivedExperimentIds.has(result.experimentId),);+ const retained = (previous.previouslyMeasured ?? [])+ .filter(+ (measurement) => Array.isArray(measurement.experiments) && Array.isArray(measurement.results),+ )+ .map((measurement) => {+ const experiments = measurement.experiments.filter((experiment) =>+ supersededExperimentIds.has(experiment.id),+ );+ const experimentIds = new Set(experiments.map((experiment) => experiment.id));+ return {+ ...measurement,+ experiments,+ results: measurement.results.filter((result) => experimentIds.has(result.experimentId)),+ };+ })+ .filter((measurement) => measurement.experiments.length > 0);if (supersededExperiments.length === 0) return retained;return [@@ -213,7 +228,28 @@ function latestValidResult(experimentId, caseId) {}function meanRunMetrics(summaryPath, model) {- const runs = readdirSync(dirname(summaryPath), { withFileTypes: true })+ const performanceRuns = runMetrics(summaryPath);+ const performanceUsage = performanceRuns.flatMap((run) =>+ run.usage === null ? [] : [run.usage],+ );+ const result = {};+ const pricing = modelPricing[model];+ if (pricing !== undefined && performanceUsage.length > 0) {+ result.meanEstimatedListCostUsd = mean(+ performanceUsage.map((value) => priceUsage(value, pricing)),+ );+ }+ if (performanceUsage.length > 0) {+ result.meanTokenConsumption = mean(performanceUsage.map(tokenConsumption));+ }+ if (performanceRuns.length > 0) {+ result.meanToolInvocationCount = mean(performanceRuns.map((run) => run.toolInvocations));+ }+ return result;+}++function runMetrics(summaryPath) {+ return readdirSync(dirname(summaryPath), { withFileTypes: true }).filter((entry) => entry.isDirectory() && /^run-\d+$/u.test(entry.name)).flatMap((entry) => {const transcriptPath = join(dirname(summaryPath), entry.name, "transcript-raw.jsonl");@@ -221,19 +257,6 @@ function meanRunMetrics(summaryPath, model) {const raw = readFileSync(transcriptPath, "utf8");return [{ usage: extractRunUsage(raw), toolInvocations: countToolInvocations(raw) }];});- const usage = runs.flatMap((run) => (run.usage === null ? [] : [run.usage]));- const result = {};- const pricing = modelPricing[model];- if (pricing !== undefined && usage.length > 0) {- result.meanEstimatedListCostUsd = mean(usage.map((value) => priceUsage(value, pricing)));- }- if (usage.length > 0) {- result.meanTokenConsumption = mean(usage.map(tokenConsumption));- }- if (runs.length > 0) {- result.meanToolInvocationCount = mean(runs.map((run) => run.toolInvocations));- }- return result;}function mean(values) {diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjsindex 0d19c8aaa..e96c56d98 100644--- a/apps/benchmarks/scripts/export-results.test.mjs+++ b/apps/benchmarks/scripts/export-results.test.mjs@@ -40,7 +40,7 @@ test("exports missing cells without publishing private artifacts", () => {assert.equal(output.schemaVersion, 1);assert.equal(output.suite.caseCount, 7);assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);- assert.equal(output.experiments.length, 16);+ assert.equal(output.experiments.length, 18);assert.deepEqual([...new Set(output.experiments.map((experiment) => experiment.modelDisplayName))],[@@ -51,10 +51,11 @@ test("exports missing cells without publishing private artifacts", () => {"GPT-5.6 Terra","Claude Sonnet 5","GLM 5.2",+ "Claude Opus 5","Gemini 3.1 Pro Preview",],);- assert.equal(output.results.length, 112);+ assert.equal(output.results.length, 126);assert.ok(output.results.every((result) => result.status === "missing"));for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {assert.equal(raw.includes(`"${privateField}"`), false);diff --git a/apps/docs/lib/evals/benchmark-results.json b/apps/docs/lib/evals/benchmark-results.jsonindex 312ba679a..03b21ac82 100644--- a/apps/docs/lib/evals/benchmark-results.json+++ b/apps/docs/lib/evals/benchmark-results.json@@ -1,6 +1,6 @@{"schemaVersion": 1,- "generatedAt": "2026-08-28T20:19:42.291Z",+ "generatedAt": "2026-08-29T02:09:15.189Z","suite": {"eveRevision": "05b12e3beb662979143397b9e691a2d7b4417a02","caseFingerprint": "786490be9d902701f57d08b3d112ca385d49a72e5591fac741511e77d67989d2",@@ -25,19 +25,19 @@"treatment": "guided"},{- "id": "claude-fable-5-opencode--baseline",- "groupId": "claude-fable-5-opencode",+ "id": "claude-fable-5-claude-code--baseline",+ "groupId": "claude-fable-5-claude-code","model": "anthropic/claude-fable-5","modelDisplayName": "Claude Fable 5",- "harness": "OpenCode",+ "harness": "Claude Code","treatment": "baseline"},{- "id": "claude-fable-5-opencode--guided",- "groupId": "claude-fable-5-opencode",+ "id": "claude-fable-5-claude-code--guided",+ "groupId": "claude-fable-5-claude-code","model": "anthropic/claude-fable-5","modelDisplayName": "Claude Fable 5",- "harness": "OpenCode",+ "harness": "Claude Code","treatment": "guided"},{@@ -89,19 +89,19 @@"treatment": "guided"},{- "id": "claude-sonnet-5-opencode--baseline",- "groupId": "claude-sonnet-5-opencode",+ "id": "claude-sonnet-5-claude-code--baseline",+ "groupId": "claude-sonnet-5-claude-code","model": "anthropic/claude-sonnet-5","modelDisplayName": "Claude Sonnet 5",- "harness": "OpenCode",+ "harness": "Claude Code","treatment": "baseline"},{- "id": "claude-sonnet-5-opencode--guided",- "groupId": "claude-sonnet-5-opencode",+ "id": "claude-sonnet-5-claude-code--guided",+ "groupId": "claude-sonnet-5-claude-code","model": "anthropic/claude-sonnet-5","modelDisplayName": "Claude Sonnet 5",- "harness": "OpenCode",+ "harness": "Claude Code","treatment": "guided"},{@@ -120,6 +120,22 @@"harness": "OpenCode","treatment": "guided"},+ {+ "id": "claude-opus-5-claude-code--baseline",+ "groupId": "claude-opus-5-claude-code",+ "model": "anthropic/claude-opus-5",+ "modelDisplayName": "Claude Opus 5",+ "harness": "Claude Code",+ "treatment": "baseline"+ },+ {+ "id": "claude-opus-5-claude-code--guided",+ "groupId": "claude-opus-5-claude-code",+ "model": "anthropic/claude-opus-5",+ "modelDisplayName": "Claude Opus 5",+ "harness": "Claude Code",+ "treatment": "guided"+ },{"id": "gemini-3-1-pro-preview-opencode--baseline","groupId": "gemini-3-1-pro-preview-opencode",@@ -307,172 +323,172 @@"meanToolInvocationCount": 12.333333333333334},{- "experimentId": "claude-fable-5-opencode--baseline",+ "experimentId": "claude-fable-5-claude-code--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 72217,- "measuredAt": "2026-08-28T19:18:44.000Z",- "meanEstimatedListCostUsd": 0.30839,- "meanTokenConsumption": 19988.333333333332,- "meanToolInvocationCount": 12+ "meanDurationMs": 127045,+ "measuredAt": "2026-08-28T22:06:23.308Z",+ "meanEstimatedListCostUsd": 6.18823,+ "meanToolInvocationCount": 17.666666666666668,+ "meanTokenConsumption": 598539},{- "experimentId": "claude-fable-5-opencode--baseline",+ "experimentId": "claude-fable-5-claude-code--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 93899,- "measuredAt": "2026-08-28T19:25:21.000Z",- "meanEstimatedListCostUsd": 0.22944333333333336,- "meanTokenConsumption": 17757.666666666668,- "meanToolInvocationCount": 9+ "meanDurationMs": 169780,+ "measuredAt": "2026-08-28T22:06:23.319Z",+ "meanEstimatedListCostUsd": 3.83675,+ "meanToolInvocationCount": 8.666666666666666,+ "meanTokenConsumption": 375135},{- "experimentId": "claude-fable-5-opencode--baseline",+ "experimentId": "claude-fable-5-claude-code--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 80987,- "measuredAt": "2026-08-28T19:25:21.000Z",- "meanEstimatedListCostUsd": 0.3146933333333333,- "meanTokenConsumption": 23109.333333333332,- "meanToolInvocationCount": 16.666666666666668+ "meanDurationMs": 258551,+ "measuredAt": "2026-08-28T22:06:23.327Z",+ "meanEstimatedListCostUsd": 5.43717,+ "meanToolInvocationCount": 12.666666666666666,+ "meanTokenConsumption": 528873},{- "experimentId": "claude-fable-5-opencode--baseline",+ "experimentId": "claude-fable-5-claude-code--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 62158,- "measuredAt": "2026-08-28T19:18:44.000Z",- "meanEstimatedListCostUsd": 0.22673666666666667,- "meanTokenConsumption": 14869.666666666666,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 89365,+ "measuredAt": "2026-08-28T22:06:23.334Z",+ "meanEstimatedListCostUsd": 3.97663,+ "meanToolInvocationCount": 11.666666666666666,+ "meanTokenConsumption": 383479},{- "experimentId": "claude-fable-5-opencode--baseline",+ "experimentId": "claude-fable-5-claude-code--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 55663,- "measuredAt": "2026-08-28T19:18:44.000Z",- "meanEstimatedListCostUsd": 0.30195666666666665,- "meanTokenConsumption": 24389,- "meanToolInvocationCount": 11.666666666666666+ "meanDurationMs": 123612,+ "measuredAt": "2026-08-28T22:06:23.339Z",+ "meanEstimatedListCostUsd": 5.19217,+ "meanToolInvocationCount": 17.666666666666668,+ "meanTokenConsumption": 507153},{- "experimentId": "claude-fable-5-opencode--baseline",+ "experimentId": "claude-fable-5-claude-code--baseline","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 53720,- "measuredAt": "2026-08-28T19:18:44.000Z",- "meanEstimatedListCostUsd": 0.2800366666666667,- "meanTokenConsumption": 23315.666666666668,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 104483,+ "measuredAt": "2026-08-28T22:06:23.346Z",+ "meanEstimatedListCostUsd": 7.76357,+ "meanToolInvocationCount": 15.666666666666666,+ "meanTokenConsumption": 761257},{- "experimentId": "claude-fable-5-opencode--baseline",+ "experimentId": "claude-fable-5-claude-code--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 53286,- "measuredAt": "2026-08-28T19:18:44.000Z",- "meanEstimatedListCostUsd": 0.2505233333333334,- "meanTokenConsumption": 19769.666666666668,- "meanToolInvocationCount": 11+ "meanDurationMs": 156463,+ "measuredAt": "2026-08-28T22:06:23.352Z",+ "meanEstimatedListCostUsd": 3.47642,+ "meanToolInvocationCount": 23.666666666666668,+ "meanTokenConsumption": 339290},{- "experimentId": "claude-fable-5-opencode--guided",+ "experimentId": "claude-fable-5-claude-code--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 53166,- "measuredAt": "2026-08-28T19:18:27.000Z",- "meanEstimatedListCostUsd": 0.3223233333333333,- "meanTokenConsumption": 23616.333333333332,- "meanToolInvocationCount": 7.333333333333333+ "meanDurationMs": 80508,+ "measuredAt": "2026-08-28T22:36:19.362Z",+ "meanEstimatedListCostUsd": 3.87527,+ "meanToolInvocationCount": 7.666666666666667,+ "meanTokenConsumption": 375115},{- "experimentId": "claude-fable-5-opencode--guided",+ "experimentId": "claude-fable-5-claude-code--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 85836,- "measuredAt": "2026-08-28T19:25:03.000Z",- "meanEstimatedListCostUsd": 0.24718666666666667,- "meanTokenConsumption": 19625.333333333332,- "meanToolInvocationCount": 8.333333333333334+ "meanDurationMs": 80206,+ "measuredAt": "2026-08-28T22:36:19.376Z",+ "meanEstimatedListCostUsd": 3.40311,+ "meanToolInvocationCount": 9,+ "meanTokenConsumption": 332259},{- "experimentId": "claude-fable-5-opencode--guided",+ "experimentId": "claude-fable-5-claude-code--guided","caseId": "author-003-openapi-connection","status": "current",- "passedRuns": 0,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 277471,- "measuredAt": "2026-08-28T19:18:27.000Z",- "meanEstimatedListCostUsd": 0.4114,- "meanTokenConsumption": 19754.666666666668,- "meanToolInvocationCount": 7.666666666666667+ "meanDurationMs": 71462,+ "measuredAt": "2026-08-28T22:36:19.386Z",+ "meanEstimatedListCostUsd": 3.58212,+ "meanToolInvocationCount": 8.666666666666666,+ "meanTokenConsumption": 350688},{- "experimentId": "claude-fable-5-opencode--guided",+ "experimentId": "claude-fable-5-claude-code--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 70438,- "measuredAt": "2026-08-28T19:18:27.000Z",- "meanEstimatedListCostUsd": 0.25587,- "meanTokenConsumption": 16792.333333333332,- "meanToolInvocationCount": 11.333333333333334+ "meanDurationMs": 87034,+ "measuredAt": "2026-08-28T22:36:19.391Z",+ "meanEstimatedListCostUsd": 4.10959,+ "meanToolInvocationCount": 7.666666666666667,+ "meanTokenConsumption": 397447},{- "experimentId": "claude-fable-5-opencode--guided",+ "experimentId": "claude-fable-5-claude-code--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 41451,- "measuredAt": "2026-08-28T19:18:27.000Z",- "meanEstimatedListCostUsd": 0.29036,- "meanTokenConsumption": 24874.666666666668,- "meanToolInvocationCount": 7+ "meanDurationMs": 59604,+ "measuredAt": "2026-08-28T22:36:19.396Z",+ "meanEstimatedListCostUsd": 3.38543,+ "meanToolInvocationCount": 7,+ "meanTokenConsumption": 331919},{- "experimentId": "claude-fable-5-opencode--guided",+ "experimentId": "claude-fable-5-claude-code--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 39958,- "measuredAt": "2026-08-28T19:18:27.000Z",- "meanEstimatedListCostUsd": 0.21587,- "meanTokenConsumption": 18063,- "meanToolInvocationCount": 8+ "meanDurationMs": 64829,+ "measuredAt": "2026-08-28T22:36:19.402Z",+ "meanEstimatedListCostUsd": 3.81826,+ "meanToolInvocationCount": 7,+ "meanTokenConsumption": 374610},{- "experimentId": "claude-fable-5-opencode--guided",+ "experimentId": "claude-fable-5-claude-code--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 36170,- "measuredAt": "2026-08-28T19:18:27.000Z",- "meanEstimatedListCostUsd": 0.16552,- "meanTokenConsumption": 13610.666666666666,- "meanToolInvocationCount": 6.333333333333333+ "meanDurationMs": 71055,+ "measuredAt": "2026-08-28T22:36:19.407Z",+ "meanEstimatedListCostUsd": 3.07521,+ "meanToolInvocationCount": 7,+ "meanTokenConsumption": 299897},{"experimentId": "grok-4-6-opencode--baseline",@@ -979,172 +995,172 @@"meanToolInvocationCount": 11.666666666666666},{- "experimentId": "claude-sonnet-5-opencode--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 84504,- "measuredAt": "2026-08-28T19:20:33.000Z",- "meanEstimatedListCostUsd": 0.087304,- "meanTokenConsumption": 30901.333333333332,- "meanToolInvocationCount": 17.666666666666668+ "meanDurationMs": 105859,+ "measuredAt": "2026-08-28T22:44:03.383Z",+ "meanEstimatedListCostUsd": 1.735734,+ "meanToolInvocationCount": 18,+ "meanTokenConsumption": 851675},{- "experimentId": "claude-sonnet-5-opencode--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 85287,- "measuredAt": "2026-08-28T19:27:58.000Z",- "meanEstimatedListCostUsd": 0.047190666666666665,- "meanTokenConsumption": 16207.333333333334,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 73537,+ "measuredAt": "2026-08-28T22:44:03.395Z",+ "meanEstimatedListCostUsd": 1.07418,+ "meanToolInvocationCount": 11,+ "meanTokenConsumption": 530522},{- "experimentId": "claude-sonnet-5-opencode--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 178438,- "measuredAt": "2026-08-28T19:27:58.000Z",- "meanEstimatedListCostUsd": 0.23833933333333335,- "meanTokenConsumption": 86543,- "meanToolInvocationCount": 37+ "meanDurationMs": 78931,+ "measuredAt": "2026-08-28T22:44:03.405Z",+ "meanEstimatedListCostUsd": 1.59483,+ "meanToolInvocationCount": 18,+ "meanTokenConsumption": 789511},{- "experimentId": "claude-sonnet-5-opencode--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 69785,- "measuredAt": "2026-08-28T19:20:33.000Z",- "meanEstimatedListCostUsd": 0.05839,- "meanTokenConsumption": 16595,- "meanToolInvocationCount": 14+ "meanDurationMs": 71745,+ "measuredAt": "2026-08-28T22:44:03.411Z",+ "meanEstimatedListCostUsd": 1.021194,+ "meanToolInvocationCount": 11,+ "meanTokenConsumption": 499669},{- "experimentId": "claude-sonnet-5-opencode--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 69671,- "measuredAt": "2026-08-28T19:20:33.000Z",- "meanEstimatedListCostUsd": 0.08692,- "meanTokenConsumption": 33568,- "meanToolInvocationCount": 15.666666666666666+ "meanDurationMs": 93642,+ "measuredAt": "2026-08-28T22:44:03.416Z",+ "meanEstimatedListCostUsd": 1.82048,+ "meanToolInvocationCount": 17.666666666666668,+ "meanTokenConsumption": 901060},{- "experimentId": "claude-sonnet-5-opencode--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-006-custom-channel","status": "current",- "passedRuns": 1,+ "passedRuns": 3,"validRuns": 3,- "meanDurationMs": 352409,- "measuredAt": "2026-08-28T19:20:33.000Z",- "meanEstimatedListCostUsd": 0.017415333333333335,- "meanTokenConsumption": 5797,- "meanToolInvocationCount": 6+ "meanDurationMs": 91912,+ "measuredAt": "2026-08-28T22:44:03.422Z",+ "meanEstimatedListCostUsd": 1.172788,+ "meanToolInvocationCount": 17.333333333333332,+ "meanTokenConsumption": 578826},{- "experimentId": "claude-sonnet-5-opencode--baseline",+ "experimentId": "claude-sonnet-5-claude-code--baseline","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 69696,- "measuredAt": "2026-08-28T19:20:33.000Z",- "meanEstimatedListCostUsd": 0.06399133333333333,- "meanTokenConsumption": 21950.333333333332,- "meanToolInvocationCount": 14+ "meanDurationMs": 84747,+ "measuredAt": "2026-08-28T22:44:03.428Z",+ "meanEstimatedListCostUsd": 1.011918,+ "meanToolInvocationCount": 13.333333333333334,+ "meanTokenConsumption": 498895},{- "experimentId": "claude-sonnet-5-opencode--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-001-weather-tool","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 72487,- "measuredAt": "2026-08-28T19:19:54.000Z",- "meanEstimatedListCostUsd": 0.06199066666666666,- "meanTokenConsumption": 20955.333333333332,- "meanToolInvocationCount": 12.333333333333334+ "meanDurationMs": 69779,+ "measuredAt": "2026-08-28T23:16:01.692Z",+ "meanEstimatedListCostUsd": 1.629858,+ "meanToolInvocationCount": 12,+ "meanTokenConsumption": 801201},{- "experimentId": "claude-sonnet-5-opencode--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-002-new-project","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 97671,- "measuredAt": "2026-08-28T19:25:30.000Z",- "meanEstimatedListCostUsd": 0.037938000000000006,- "meanTokenConsumption": 11189,- "meanToolInvocationCount": 13.666666666666666+ "meanDurationMs": 56970,+ "measuredAt": "2026-08-28T23:16:01.703Z",+ "meanEstimatedListCostUsd": 0.965568,+ "meanToolInvocationCount": 7.666666666666667,+ "meanTokenConsumption": 476704},{- "experimentId": "claude-sonnet-5-opencode--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-003-openapi-connection","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 53872,- "measuredAt": "2026-08-28T19:25:30.000Z",- "meanEstimatedListCostUsd": 0.06860866666666667,- "meanTokenConsumption": 27953.666666666668,- "meanToolInvocationCount": 12+ "meanDurationMs": 42652,+ "measuredAt": "2026-08-28T23:16:01.711Z",+ "meanEstimatedListCostUsd": 0.82867,+ "meanToolInvocationCount": 8.333333333333334,+ "meanTokenConsumption": 410267},{- "experimentId": "claude-sonnet-5-opencode--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-004-packaged-skill","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 54347,- "measuredAt": "2026-08-28T19:19:54.000Z",- "meanEstimatedListCostUsd": 0.05166133333333334,- "meanTokenConsumption": 17134.666666666668,- "meanToolInvocationCount": 10.333333333333334+ "meanDurationMs": 45265,+ "measuredAt": "2026-08-28T23:16:01.717Z",+ "meanEstimatedListCostUsd": 0.67903,+ "meanToolInvocationCount": 5,+ "meanTokenConsumption": 332683},{- "experimentId": "claude-sonnet-5-opencode--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-005-conditional-approval","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 57140,- "measuredAt": "2026-08-28T19:19:54.000Z",- "meanEstimatedListCostUsd": 0.04402533333333333,- "meanTokenConsumption": 15630,- "meanToolInvocationCount": 12+ "meanDurationMs": 39690,+ "measuredAt": "2026-08-28T23:16:01.722Z",+ "meanEstimatedListCostUsd": 0.972688,+ "meanToolInvocationCount": 7,+ "meanTokenConsumption": 480972},{- "experimentId": "claude-sonnet-5-opencode--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-006-custom-channel","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 46268,- "measuredAt": "2026-08-28T19:19:54.000Z",- "meanEstimatedListCostUsd": 0.026169333333333333,- "meanTokenConsumption": 8867.333333333334,- "meanToolInvocationCount": 8+ "meanDurationMs": 38868,+ "measuredAt": "2026-08-28T23:16:01.727Z",+ "meanEstimatedListCostUsd": 1.219158,+ "meanToolInvocationCount": 7.333333333333333,+ "meanTokenConsumption": 604483},{- "experimentId": "claude-sonnet-5-opencode--guided",+ "experimentId": "claude-sonnet-5-claude-code--guided","caseId": "author-007-digest-schedule","status": "current","passedRuns": 3,"validRuns": 3,- "meanDurationMs": 27822,- "measuredAt": "2026-08-28T19:19:54.000Z",- "meanEstimatedListCostUsd": 0.01684333333333333,- "meanTokenConsumption": 5760.333333333333,- "meanToolInvocationCount": 6+ "meanDurationMs": 68924,+ "measuredAt": "2026-08-28T23:16:01.731Z",+ "meanEstimatedListCostUsd": 1.053378,+ "meanToolInvocationCount": 15,+ "meanTokenConsumption": 521965},{"experimentId": "glm-5-2-opencode--baseline",@@ -1314,6 +1330,174 @@"meanTokenConsumption": 16295.666666666666,"meanToolInvocationCount": 12.333333333333334},+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 135534,+ "measuredAt": "2026-08-29T00:05:25.109Z",+ "meanEstimatedListCostUsd": 4.933595,+ "meanToolInvocationCount": 22.666666666666668,+ "meanTokenConsumption": 947443+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 92582,+ "measuredAt": "2026-08-29T00:05:25.121Z",+ "meanEstimatedListCostUsd": 2.50535,+ "meanToolInvocationCount": 13.333333333333334,+ "meanTokenConsumption": 488330+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 191566,+ "measuredAt": "2026-08-29T00:05:25.129Z",+ "meanEstimatedListCostUsd": 14.008915,+ "meanToolInvocationCount": 36.333333333333336,+ "meanTokenConsumption": 2751467+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 125522,+ "measuredAt": "2026-08-29T00:05:25.138Z",+ "meanEstimatedListCostUsd": 3.74517,+ "meanToolInvocationCount": 16,+ "meanTokenConsumption": 722970+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 135687,+ "measuredAt": "2026-08-29T00:05:25.146Z",+ "meanEstimatedListCostUsd": 4.87658,+ "meanToolInvocationCount": 26.666666666666668,+ "meanTokenConsumption": 952132+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 202636,+ "measuredAt": "2026-08-29T00:05:25.152Z",+ "meanToolInvocationCount": 26.333333333333332,+ "meanTokenConsumption": 1822465,+ "meanEstimatedListCostUsd": 9.304045+ },+ {+ "experimentId": "claude-opus-5-claude-code--baseline",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 163050,+ "measuredAt": "2026-08-29T00:05:25.160Z",+ "meanEstimatedListCostUsd": 5.362835,+ "meanToolInvocationCount": 26,+ "meanTokenConsumption": 1044543+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-001-weather-tool",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 106811,+ "measuredAt": "2026-08-29T00:13:01.331Z",+ "meanEstimatedListCostUsd": 11.438505,+ "meanToolInvocationCount": 14.666666666666666,+ "meanTokenConsumption": 2234333+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-002-new-project",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 91787,+ "measuredAt": "2026-08-29T00:13:01.344Z",+ "meanEstimatedListCostUsd": 2.4951,+ "meanToolInvocationCount": 16,+ "meanTokenConsumption": 486252+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-003-openapi-connection",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 114352,+ "measuredAt": "2026-08-29T00:13:01.356Z",+ "meanEstimatedListCostUsd": 2.90723,+ "meanToolInvocationCount": 20.666666666666668,+ "meanTokenConsumption": 568682+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-004-packaged-skill",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 77606,+ "measuredAt": "2026-08-29T00:13:01.363Z",+ "meanEstimatedListCostUsd": 2.328845,+ "meanToolInvocationCount": 10,+ "meanTokenConsumption": 446685+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-005-conditional-approval",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 104699,+ "measuredAt": "2026-08-29T00:13:01.370Z",+ "meanEstimatedListCostUsd": 4.55151,+ "meanToolInvocationCount": 19,+ "meanTokenConsumption": 888618+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-006-custom-channel",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 162364,+ "measuredAt": "2026-08-29T00:13:01.377Z",+ "meanEstimatedListCostUsd": 6.511585,+ "meanToolInvocationCount": 16.333333333333332,+ "meanTokenConsumption": 1266737+ },+ {+ "experimentId": "claude-opus-5-claude-code--guided",+ "caseId": "author-007-digest-schedule",+ "status": "current",+ "passedRuns": 3,+ "validRuns": 3,+ "meanDurationMs": 89702,+ "measuredAt": "2026-08-29T00:13:01.384Z",+ "meanEstimatedListCostUsd": 3.44083,+ "meanToolInvocationCount": 16.333333333333332,+ "meanTokenConsumption": 674858+ },{"experimentId": "gemini-3-1-pro-preview-opencode--baseline","caseId": "author-001-weather-tool",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 4f0eb8462..f89d9f081 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -139,20 +139,23 @@ importers:apps/benchmarks:dependencies:'@ai-sdk/harness':- specifier: 1.0.72+ specifier: 1.0.87+ '@ai-sdk/harness-claude-code':+ specifier: 1.0.90+ version: 1.0.90([email protected])([email protected])'@ai-sdk/harness-opencode':- specifier: 1.0.73- version: 1.0.73([email protected])([email protected])+ specifier: 1.0.88+ version: 1.0.88([email protected])([email protected])'@ai-sdk/sandbox-vercel':- specifier: 1.0.72+ specifier: 1.0.87'@vercel/agent-eval':specifier: 1.5.0version: 1.5.0([email protected])'@vercel/sandbox':- specifier: 'catalog:'- version: 2.8.0+ specifier: 3.2.0+ version: 3.2.0jiti:specifier: 2.7.0version: 2.7.0@@ -253,10 +256,10 @@ importers:version: 0.33.0(@vercel/[email protected])(eve@packages+eve)'@agentphone/chat-sdk-adapter':specifier: 0.1.0- version: 0.1.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])+ version: 0.1.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])'@beeper/chat-adapter-matrix':specifier: 0.2.0- version: 0.2.0(@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])+ version: 0.2.0(@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])'@blitzreels/eve':specifier: ^0.2.1version: 0.2.1(eve@packages+eve)@@ -265,49 +268,49 @@ importers:version: 0.1.0(@cfworker/[email protected])(eve@packages+eve)([email protected])'@chat-adapter/gchat':specifier: 4.34.0'@chat-adapter/messenger':specifier: 4.34.0'@chat-adapter/state-memory':specifier: 4.34.0'@chat-adapter/whatsapp':specifier: 4.34.0'@chat-adapter/x':specifier: 4.34.0'@eve/self-modification':specifier: workspace:*version: link:../../packages/eve-self-modification'@getdial/chat-sdk-adapter':specifier: 0.2.0- version: 0.2.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected](@babel/[email protected]([email protected]))(@types/[email protected])([email protected])([email protected])([email protected]))([email protected])([email protected])+ version: 0.2.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected](@babel/[email protected]([email protected]))(@types/[email protected])([email protected])([email protected])([email protected]))([email protected])([email protected])'@github-tools/eve-extension':specifier: ^0.1.0- version: 0.1.0(@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)+ version: 0.1.0(@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)'@jetty/eve':specifier: ^0.3.0version: 0.3.0(eve@packages+eve)'@kapso/chat-adapter':specifier: 0.1.1- version: 0.1.1([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])+ version: 0.1.1([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])'@kybernesis/arcana':specifier: 0.2.0version: 0.2.0(eve@packages+eve)'@larksuite/vercel-chat-adapter':specifier: 0.1.2- version: 0.1.2([email protected]([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])+ version: 0.1.2([email protected]([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])'@liveblocks/chat-sdk-adapter':specifier: 3.23.0- version: 3.23.0(@types/[email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))+ version: 3.23.0(@types/[email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))'@novu/chat-sdk-adapter':specifier: 0.1.0- version: 0.1.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])([email protected])+ version: 0.1.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])([email protected])'@onkernel/eve-extension':specifier: ^0.1.3- version: 0.1.3(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)+ version: 0.1.3(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)'@opentelemetry/api':specifier: 1.9.1version: 1.9.1@@ -322,7 +325,7 @@ importers:version: 7.19.6(@ai-sdk/[email protected])(@modelcontextprotocol/[email protected](@cfworker/[email protected])([email protected])([email protected]))(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected](@typescript-eslint/[email protected]))([email protected]([email protected]))([email protected]([email protected]))'@resend/chat-sdk-adapter':specifier: 0.2.2- version: 0.2.2(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]))(@react-email/[email protected]([email protected]([email protected]))([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])+ version: 0.2.2(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]))(@react-email/[email protected]([email protected]([email protected]))([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])'@tailwindcss/postcss':specifier: 4.3.0version: 4.3.0@@ -352,10 +355,10 @@ importers:version: 0.2.1(eve@packages+eve)'@veltdev/chat-sdk-adapter':specifier: 0.2.4- version: 0.2.4([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])+ version: 0.2.4([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])'@vercel/connect':specifier: 'catalog:'- version: 1.0.0(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)+ version: 1.0.0(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)'@vercel/otel':specifier: 'catalog:'version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))@@ -367,22 +370,22 @@ importers:version: 0.1.71'@zernio/chat-sdk-adapter':specifier: 0.5.0- version: 0.5.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])+ version: 0.5.0([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])braintrust:specifier: 3.24.0chat:specifier: 4.34.0chat-adapter-sendblue:specifier: 0.2.0- version: 0.2.0([email protected]([email protected]([email protected]))([email protected])([email protected]))+ version: 0.2.0([email protected]([email protected]([email protected]))([email protected])([email protected]))eve:specifier: workspace:*version: link:../../packages/eveeve-channel-blooio:specifier: 0.2.0- version: 0.2.0([email protected]([email protected]))(eve@packages+eve)+ version: 0.2.0([email protected]([email protected]))(eve@packages+eve)postcss:specifier: 8.5.15version: 8.5.15@@ -690,7 +693,7 @@ importers:version: 25.9.1historical-eve-0-30-8:specifier: npm:[email protected]- version: [email protected](@opentelemetry/[email protected])(@upstash/[email protected])([email protected]([email protected]))([email protected](@aws-sdk/[email protected])([email protected])([email protected]))([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))+ version: [email protected](@opentelemetry/[email protected])(@upstash/[email protected])([email protected]([email protected]))([email protected](@aws-sdk/[email protected])([email protected])([email protected]))([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))typescript:specifier: 'catalog:'version: 7.0.2@@ -1619,8 +1622,8 @@ packages:peerDependencies:zod: ^3.25.76 || ^4.1.8- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-SXUM8jzzuTUJRq+EOgPd5to6DSx0EKslVn+IVZHbUEX6k/3vCPNrvjckbK26HnNxHU/STxm+zTSJteqrO+7Z0w==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-iayK1z11I0b3sgbP3j6eKQypfpwO5Fbtaq1Mkwv7YxwGQG1YTNr5QAjyYsSfqyf/58WUOLWz2cTBpk+ExZVgpw==}engines: {node: '>=22'}peerDependencies:zod: ^3.25.76 || ^4.1.8@@ -1649,14 +1652,20 @@ packages:peerDependencies:zod: ^3.25.76 || ^4.1.8- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-OewZBK5NuufxY9Ve9VahtQ2ma3oaiZXO89iXB5rmzyyF4kMCx5nDNirAwY/HaR8rcwhUE4gGGuoBjNUNPHkaYA==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-SL77y5l3596n6cxuLN4OB19iTkvwg6zJq2KxIV9MAPwAMYEEXHz7YGRXxrAd7GOixIVqcsXEw1HOLN1zcG1L3A==}+ engines: {node: '>=22'}+ peerDependencies:+ zod: ^3.25.76 || ^4.1.8++ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-bA22/rBaTbsO/o/x91P78RvLpvgwnCoW6Ie7ATsC7Z6BxGYZpFrS6BGWD1DQL2pN3jy61KWnDLDwy0egg7ypZg==}engines: {node: '>=22'}peerDependencies:zod: ^3.25.76 || ^4.1.8- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-p11kq04JlKCy6VCjzhyeGZJhQbdYkGYdkCOLspFVoBYoW7hp2Co6nob8o2l1uRAsGTFqGdWZ6ZFaKov1K+BjKA==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-OY+M1B58lLv+fRHB0dh/FJ6lCzmg9HQmTe1z+XS1WTPF8PnTMEr2E/6vNTUoORN7iZMkTyVkszA4DOhod1eFQw==}engines: {node: '>=22'}peerDependencies:ws: ^8.21.0@@ -1729,8 +1738,8 @@ packages:peerDependencies:zod: ^3.25.76 || ^4.1.8- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-EzAn4pdgG5g0xXtH6lE2zyNmfjDQIDjATkfqzuidEI35g++hh4+07vnjzkT/RmGmIClPZiRj/Q2GMPV2V7mkHw==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-9TyxUXolql77ntHIeygLqt7PO1O0HZPB73cJhLG2OsPecSBRIZhNXLxT1T7rlL6Zpm1eDoLMFrMV99kAJ28/Sg==}engines: {node: '>=22'}peerDependencies:zod: ^3.25.76 || ^4.1.8@@ -1751,14 +1760,18 @@ packages:resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==}engines: {node: '>=22'}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-aWO7iwhFUGf347tCwNGggggfmZigaSu7TF739IZSrWWABUp7zkb4Cr3fMqvBe5EIS7ABJJu3Cadn0g/zs1G0QQ==}+ engines: {node: '>=22'}+'@ai-sdk/[email protected]':resolution: {integrity: sha512-El0jUZ/B7mvBHAD5rfSDqOAhWxutVTq7BCNhfGuwfDPT9SO0TMHybh2bMkieJQI7YOfl+qNBoWrRAOHHaFb99Q==}engines: {node: '>=18'}peerDependencies:react: ^18 || ~19.0.1 || ~19.1.2 || ^19.2.1- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-AolOfKEZ5VhsEXRVwDGroabY5Fy2fCNxEFFYpdjqd4J/1IxgpBSgoYNWOS0T1Xg/0J4WCccAM7PQwpmJgwLtVw==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-rW8mtwvH5lcoXWPn8hEdh2Kns/2qAQokS9xG/UeqZOgp79ifKJQp8dvWpUSzRNZKgPtqsXNmVFac6J/UnlmO+Q==}engines: {node: '>=22'}'@ai-sdk/[email protected]':@@ -8495,8 +8508,8 @@ packages:peerDependencies:zod: ^3.25.76 || ^4.1.8- resolution: {integrity: sha512-wBUyoCYF3GVr+62nelBgR8YbpTSsMZrzFyOOjiwijylNSM2TFCW35C+Pml2vc59/WLMpyhS/LWZ55M+B9DAcSg==}+ resolution: {integrity: sha512-zZkSUq6MgmXjwIfML2wkPFeQ5Azk8kGEWkciAtE0NgnFy3slj4ljsw5LvigsCbYXxTSVr+6iCHBTPZafpx1P0w==}engines: {node: '>=22'}peerDependencies:zod: ^3.25.76 || ^4.1.8@@ -15921,10 +15934,10 @@ snapshots:dependencies:zod: 4.4.3- '@agentphone/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':+ '@agentphone/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])transitivePeerDependencies:- ai- supports-color@@ -16012,10 +16025,10 @@ snapshots:'@vercel/oidc': 3.2.0zod: 4.4.3- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 4.0.7- '@ai-sdk/provider-utils': 5.0.27([email protected])+ '@ai-sdk/provider': 4.0.8+ '@ai-sdk/provider-utils': 5.0.30([email protected])'@vercel/oidc': 3.2.0zod: 4.4.3@@ -16052,25 +16065,44 @@ snapshots:zod: 4.4.3optional: truedependencies:- '@ai-sdk/provider-utils': 5.0.27([email protected])+ '@ai-sdk/provider-utils': 5.0.30([email protected])+ ws: 8.21.3([email protected])+ zod: 4.4.3+ transitivePeerDependencies:+ - bufferutil+ - utf-8-validate++ dependencies:+ '@ai-sdk/provider-utils': 5.0.30([email protected])ws: 8.21.0([email protected])zod: 4.4.3transitivePeerDependencies:- bufferutil- utf-8-validatedependencies:- '@ai-sdk/provider': 4.0.7- '@ai-sdk/provider-utils': 5.0.27([email protected])- ai: 7.0.66([email protected])+ '@ai-sdk/provider': 4.0.8+ '@ai-sdk/provider-utils': 5.0.30([email protected])+ ai: 7.0.79([email protected])zod: 4.4.3optionalDependencies:ws: 8.21.0([email protected])+ dependencies:+ '@ai-sdk/provider': 4.0.8+ '@ai-sdk/provider-utils': 5.0.30([email protected])+ ai: 7.0.79([email protected])+ zod: 4.4.3+ optionalDependencies:+ ws: 8.21.3([email protected])+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 4.0.7@@ -16157,9 +16189,9 @@ snapshots:undici: 7.29.0zod: 4.4.3- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 4.0.7+ '@ai-sdk/provider': 4.0.8'@standard-schema/spec': 1.1.0'@workflow/serde': 4.1.0eventsource-parser: 3.1.0@@ -16182,6 +16214,10 @@ snapshots:dependencies:json-schema: 0.4.0+ '@ai-sdk/[email protected]':+ dependencies:+ json-schema: 0.4.0+dependencies:'@ai-sdk/provider-utils': 4.0.27([email protected])@@ -16192,11 +16228,11 @@ snapshots:transitivePeerDependencies:- zoddependencies:- '@ai-sdk/provider-utils': 5.0.27([email protected])- '@vercel/sandbox': 2.8.0+ '@ai-sdk/provider-utils': 5.0.30([email protected])+ '@vercel/sandbox': 3.2.0transitivePeerDependencies:- bare-abort-controller- react-native-b4a@@ -16532,11 +16568,11 @@ snapshots:'@balena/[email protected]': {}- '@beeper/[email protected](@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])':+ '@beeper/[email protected](@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/state-memory': 4.34.0([email protected]([email protected]))([email protected])([email protected])- '@chat-adapter/state-redis': 4.35.0(@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/state-memory': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/state-redis': 4.35.0(@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])marked: 15.0.12matrix-js-sdk: 41.9.0node-html-parser: 7.1.0@@ -16786,29 +16822,29 @@ snapshots:human-id: 4.1.3prettier: 2.8.8- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])'@googleapis/chat': 44.7.0([email protected])'@googleapis/workspaceevents': 9.2.0([email protected])transitivePeerDependencies:- ai- supports-color- zod- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])transitivePeerDependencies:- ai- supports-color- zod- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':dependencies:transitivePeerDependencies:- ai- supports-color@@ -16822,9 +16858,9 @@ snapshots:- supports-color- zod- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':dependencies:transitivePeerDependencies:- ai- supports-color@@ -16859,12 +16895,12 @@ snapshots:- zodoptional: true- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])'@slack/socket-mode': 2.0.7([email protected])([email protected]([email protected]))([email protected])transitivePeerDependencies:- ai- bufferutil@@ -16882,17 +16918,17 @@ snapshots:- supports-color- zod- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':dependencies:transitivePeerDependencies:- ai- supports-color- zod- '@chat-adapter/[email protected](@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected](@opentelemetry/[email protected])([email protected]([email protected]))([email protected])([email protected])':dependencies:redis: 5.12.1(@opentelemetry/[email protected])transitivePeerDependencies:- '@node-rs/xxhash'@@ -16911,19 +16947,19 @@ snapshots:- supports-color- zod- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])transitivePeerDependencies:- ai- supports-color- zod- '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':+ '@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])transitivePeerDependencies:- ai- supports-color@@ -17414,11 +17450,11 @@ snapshots:dependencies:tslib: 2.8.1- '@getdial/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected](@babel/[email protected]([email protected]))(@types/[email protected])([email protected])([email protected])([email protected]))([email protected])([email protected])':+ '@getdial/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected](@babel/[email protected]([email protected]))(@types/[email protected])([email protected])([email protected])([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])'@getdial/sdk': 0.19.1([email protected](@babel/[email protected]([email protected]))(@types/[email protected])([email protected])([email protected])([email protected]))([email protected])transitivePeerDependencies:- ai- encoding@@ -17435,26 +17471,26 @@ snapshots:- react-native- supports-color- '@github-tools/[email protected](@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)':+ '@github-tools/[email protected](@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)':dependencies:- '@github-tools/sdk': 1.8.2(@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)([email protected])+ '@github-tools/sdk': 1.8.2(@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)([email protected])eve: link:packages/evezod: 4.4.3optionalDependencies:- '@vercel/connect': 1.0.0(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)+ '@vercel/connect': 1.0.0(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)transitivePeerDependencies:- '@ai-sdk/workflow'- '@workflow/ai'- ai- workflow- '@github-tools/[email protected](@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)([email protected])':+ '@github-tools/[email protected](@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve))([email protected]([email protected]))(eve@packages+eve)([email protected])':dependencies:- ai: 7.0.66([email protected])+ ai: 7.0.79([email protected])octokit: 5.0.5zod: 4.4.3optionalDependencies:- '@vercel/connect': 1.0.0(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)+ '@vercel/connect': 1.0.0(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)eve: link:packages/eve'@google/[email protected](@modelcontextprotocol/[email protected](@cfworker/[email protected])([email protected])([email protected]))([email protected])([email protected])':@@ -17833,11 +17869,11 @@ snapshots:'@js-sdsl/[email protected]': {}- '@kapso/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':+ '@kapso/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])'@kapso/whatsapp-cloud-api': 0.2.3transitivePeerDependencies:- ai- supports-color@@ -17909,11 +17945,11 @@ snapshots:'@langchain/[email protected]': {}- '@larksuite/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])':+ '@larksuite/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])transitivePeerDependencies:- ai- bufferutil@@ -17946,11 +17982,11 @@ snapshots:dependencies:standardwebhooks: 1.0.0- '@liveblocks/[email protected](@types/[email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))':+ '@liveblocks/[email protected](@types/[email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))':dependencies:'@liveblocks/core': 3.23.0(@types/[email protected])'@liveblocks/node': 3.23.0(@types/[email protected])transitivePeerDependencies:- '@types/json-schema'- encoding@@ -18308,10 +18344,10 @@ snapshots:'@nolyfill/[email protected]': {}- '@novu/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])([email protected])':+ '@novu/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.30.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.30.0([email protected]([email protected]))([email protected])([email protected])optionalDependencies:react: 19.2.6transitivePeerDependencies:@@ -18942,9 +18978,9 @@ snapshots:'@octokit/request-error': 7.1.0'@octokit/webhooks-methods': 6.0.0- '@onkernel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)':+ '@onkernel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)':dependencies:- '@vercel/connect': 0.4.3(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)+ '@vercel/connect': 0.4.3(@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)eve: link:packages/evezod: 4.4.3transitivePeerDependencies:@@ -20963,10 +20999,10 @@ snapshots:'@repeaterjs/[email protected]': {}- '@resend/[email protected](@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]))(@react-email/[email protected]([email protected]([email protected]))([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])':+ '@resend/[email protected](@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]))(@react-email/[email protected]([email protected]([email protected]))([email protected]))([email protected])([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])hast-util-to-html: 9.0.5mdast-util-to-hast: 13.2.1react-email: 6.9.1([email protected])([email protected]([email protected]))([email protected])([email protected])@@ -22270,10 +22306,10 @@ snapshots:dependencies:eve: link:packages/eve- '@veltdev/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':+ '@veltdev/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])hast-util-from-html: 2.0.3hast-util-to-html: 9.0.5hast-util-to-mdast: 10.1.2@@ -22414,14 +22450,14 @@ snapshots:dependencies:execa: 5.1.1- '@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)':+ '@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)':dependencies:'@vercel/oidc': 3.8.1optionalDependencies:'@ai-sdk/mcp': 2.0.29([email protected])'@auth/core': 0.41.2- '@chat-adapter/slack': 4.34.0([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])- ai: 7.0.66([email protected])+ '@chat-adapter/slack': 4.34.0([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])+ ai: 7.0.79([email protected])eve: link:packages/eve'@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)':@@ -22434,14 +22470,14 @@ snapshots:ai: 7.0.58([email protected])eve: link:packages/eve- '@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)':+ '@vercel/[email protected](@ai-sdk/[email protected]([email protected]))(@auth/[email protected])(@chat-adapter/[email protected]([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]))([email protected]([email protected]))(eve@packages+eve)':dependencies:'@vercel/oidc': 3.8.5optionalDependencies:'@ai-sdk/mcp': 2.0.29([email protected])'@auth/core': 0.41.2- '@chat-adapter/slack': 4.34.0([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])- ai: 7.0.66([email protected])+ '@chat-adapter/slack': 4.34.0([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])+ ai: 7.0.79([email protected])eve: link:packages/eve'@vercel/[email protected](@vercel/[email protected])':@@ -23358,10 +23394,10 @@ snapshots:ulid: 3.0.2zod: 4.3.6- '@zernio/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':+ '@zernio/[email protected]([email protected]([email protected]))([email protected]([email protected]([email protected]))([email protected])([email protected]))([email protected])([email protected])':dependencies:- '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])+ '@chat-adapter/shared': 4.34.0([email protected]([email protected]))([email protected])([email protected])transitivePeerDependencies:- ai- supports-color@@ -23442,11 +23478,11 @@ snapshots:'@ai-sdk/provider-utils': 5.0.25([email protected])zod: 4.4.3dependencies:- '@ai-sdk/gateway': 4.0.52([email protected])- '@ai-sdk/provider': 4.0.7- '@ai-sdk/provider-utils': 5.0.27([email protected])+ '@ai-sdk/gateway': 4.0.64([email protected])+ '@ai-sdk/provider': 4.0.8+ '@ai-sdk/provider-utils': 5.0.30([email protected])zod: 4.4.3@@ -24041,12 +24077,12 @@ snapshots:- [email protected]([email protected]([email protected]([email protected]))([email protected])([email protected])):+ [email protected]([email protected]([email protected]([email protected]))([email protected])([email protected])):dependencies:sendblue: 3.10.1dependencies:'@workflow/serde': 4.1.0-beta.2mdast-util-to-string: 4.0.0@@ -24056,7 +24092,7 @@ snapshots:remend: 1.3.0unified: 11.0.5optionalDependencies:- ai: 7.0.66([email protected])+ ai: 7.0.79([email protected])zod: 4.4.3transitivePeerDependencies:- supports-color@@ -24076,7 +24112,7 @@ snapshots:transitivePeerDependencies:- supports-colordependencies:'@workflow/serde': 4.1.0-beta.2mdast-util-to-string: 4.0.0@@ -24086,12 +24122,12 @@ snapshots:remend: 1.3.0unified: 11.0.5optionalDependencies:- ai: 7.0.66([email protected])+ ai: 7.0.79([email protected])zod: 4.4.3transitivePeerDependencies:- supports-colordependencies:'@workflow/serde': 4.1.0-beta.2mdast-util-to-string: 4.0.0@@ -24101,7 +24137,7 @@ snapshots:remend: 1.3.0unified: 11.0.5optionalDependencies:- ai: 7.0.66([email protected])+ ai: 7.0.79([email protected])zod: 4.4.3transitivePeerDependencies:- supports-color@@ -25577,9 +25613,9 @@ snapshots:dependencies:- ai: 7.0.66([email protected])+ ai: 7.0.79([email protected])eve: link:packages/eve[email protected](@opentelemetry/[email protected])(@upstash/[email protected])(@vercel/[email protected])(@vercel/[email protected](@aws-sdk/[email protected])([email protected]([email protected])))([email protected]([email protected]))([email protected])([email protected](@aws-sdk/[email protected])([email protected])([email protected]))([email protected])([email protected])([email protected](@opentelemetry/[email protected])(@types/[email protected])(@upstash/[email protected])([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])):@@ -25632,9 +25668,9 @@ snapshots:- xml2js- zephyr-agent- [email protected](@opentelemetry/[email protected])(@upstash/[email protected])([email protected]([email protected]))([email protected](@aws-sdk/[email protected])([email protected])([email protected]))([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])):+ [email protected](@opentelemetry/[email protected])(@upstash/[email protected])([email protected]([email protected]))([email protected](@aws-sdk/[email protected])([email protected])([email protected]))([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])):dependencies:- ai: 7.0.66([email protected])+ ai: 7.0.79([email protected])nitro: 3.0.260610-beta(@upstash/[email protected])(@vercel/[email protected])(@vercel/[email protected](@aws-sdk/[email protected])([email protected]([email protected])))([email protected])([email protected])([email protected])([email protected](@opentelemetry/[email protected])(@types/[email protected])(@upstash/[email protected])([email protected])([email protected]))([email protected])([email protected]([email protected]))([email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))undici: 8.9.0optionalDependencies: