Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

Update Braintrust registry scaffold (#1295)

Casey Gowrie782ff7732fba
diff --git a/apps/docs/lib/integrations/data.ts b/apps/docs/lib/integrations/data.ts
index c3f54ab88..6bf6cc0ef 100644
--- a/apps/docs/lib/integrations/data.ts
+++ b/apps/docs/lib/integrations/data.ts
@@ -1726,9 +1726,8 @@ const connectionPresentations: Record<string, ConnectionPresentation> = {
/**
* Instrumentation overlay: presentation plus hand-authored setup markdown.
- * Instrumentation providers are OpenTelemetry backends configured in
- * `agent/instrumentation.ts`, so they follow the channel shape (markdown)
- * rather than the generated connection shape.
+ * Instrumentation providers use hand-authored setup files, so they follow the
+ * channel shape (markdown) rather than the generated connection shape.
*/
type InstrumentationPresentation = ChannelPresentation;
@@ -1737,32 +1736,49 @@ const instrumentationPresentations: Record<string, InstrumentationPresentation>
logo: "braintrust",
docsHref: "/docs/guides/instrumentation",
keywords: ["otel", "opentelemetry", "tracing", "observability", "evals", "monitoring"],
- install: `Add Braintrust instrumentation from eve's registry:
+ install: `Add the Braintrust integration from eve's registry:
\`\`\`bash
eve add instrumentation/braintrust
\`\`\``,
- quickStart: `Create \`agent/instrumentation.ts\`. eve auto-discovers it and runs it at server startup, and its presence enables telemetry:
+ quickStart: `eve installs a hook that traces agent activity and an instrumentation file that initializes the Braintrust logger:
+
+\`\`\`ts
+// agent/hooks/braintrust.ts
+import { braintrustEveHook } from "braintrust";
+import { defineState } from "eve/context";
+import { defineHook } from "eve/hooks";
+
+export default defineHook(
+ braintrustEveHook({
+ defineState,
+ metadata: {
+ app: "my-eve-agent", // Replace with your app name
+ },
+ }) as Parameters<typeof defineHook>[0],
+);
+\`\`\`
\`\`\`ts
// agent/instrumentation.ts
-import { BraintrustExporter } from "@braintrust/otel";
+import { braintrustEveInstrumentation, initLogger } from "braintrust";
+import { defineState } from "eve/context";
import { defineInstrumentation } from "eve/instrumentation";
-import { registerOTel } from "@vercel/otel";
-export default defineInstrumentation({
- setup: ({ agentName }) =>
- registerOTel({
- serviceName: agentName,
- traceExporter: new BraintrustExporter({
- parent: \`project_name:\${agentName}\`,
- filterAISpans: true,
- }),
- }),
-});
+export default defineInstrumentation(
+ braintrustEveInstrumentation({
+ defineState,
+ setup: ({ agentName }) => {
+ initLogger({
+ projectName: agentName,
+ apiKey: process.env.BRAINTRUST_API_KEY,
+ });
+ },
+ }) as Parameters<typeof defineInstrumentation>[0],
+);
\`\`\``,
- configure: `Create an API key in the Braintrust dashboard and expose it as \`BRAINTRUST_API_KEY\`. Spans land in the Braintrust project named after your agent. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,
+ configure: `Create an API key in the Braintrust dashboard and expose it as \`BRAINTRUST_API_KEY\`. Replace the hook's \`app\` metadata with your app name. Spans land in the Braintrust project named after your agent. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,
},
"sentry-instrumentation": {
logo: "sentry",
diff --git a/apps/docs/package.json b/apps/docs/package.json
index 5b6643416..8321e9e45 100644
--- a/apps/docs/package.json
+++ b/apps/docs/package.json
@@ -67,7 +67,6 @@
"@agent-browser/eve": "^0.33.0",
"@agentphone/chat-sdk-adapter": "0.1.0",
"@beeper/chat-adapter-matrix": "0.2.0",
- "@braintrust/otel": "^0.3.0",
"@browserbasehq/eve": "^0.1.0",
"@chat-adapter/gchat": "4.34.0",
"@chat-adapter/messenger": "4.34.0",
@@ -98,6 +97,7 @@
"@vgpu/adapter-node": "0.0.7",
"@webgpu/types": "^0.1.69",
"@zernio/chat-sdk-adapter": "0.5.0",
+ "braintrust": "3.24.0",
"chat": "4.34.0",
"chat-adapter-sendblue": "0.2.0",
"eve": "workspace:*",
diff --git a/apps/docs/public/r/instrumentation/braintrust.json b/apps/docs/public/r/instrumentation/braintrust.json
index f98317e98..7b74e5757 100644
--- a/apps/docs/public/r/instrumentation/braintrust.json
+++ b/apps/docs/public/r/instrumentation/braintrust.json
@@ -2,18 +2,26 @@
"$schema": "https://ui.shadcn.com/schema/registry-item.json",
"name": "instrumentation/braintrust",
"title": "Braintrust",
- "description": "Export AI SDK spans to Braintrust for tracing, evals, and monitoring.",
- "dependencies": ["@braintrust/otel", "@vercel/otel"],
+ "description": "Trace eve agent sessions in Braintrust, including turns, steps, tool calls, and subagent interactions.",
+ "dependencies": [
+ "braintrust"
+ ],
"files": [
{
"path": "registry/instrumentation/braintrust.ts",
- "content": "import { BraintrustExporter } from \"@braintrust/otel\";\nimport { registerOTel } from \"@vercel/otel\";\nimport { defineInstrumentation } from \"eve/instrumentation\";\n\nexport default defineInstrumentation({\n setup: ({ agentName }) =>\n registerOTel({\n serviceName: agentName,\n traceExporter: new BraintrustExporter({\n parent: `project_name:${agentName}`,\n filterAISpans: true,\n }),\n }),\n});\n",
+ "content": "import { braintrustEveInstrumentation, initLogger } from \"braintrust\";\nimport { defineState } from \"eve/context\";\nimport { defineInstrumentation } from \"eve/instrumentation\";\n\nexport default defineInstrumentation(\n braintrustEveInstrumentation({\n defineState,\n setup: ({ agentName }) => {\n initLogger({\n projectName: agentName,\n apiKey: process.env.BRAINTRUST_API_KEY,\n });\n },\n }) as Parameters<typeof defineInstrumentation>[0],\n);\n",
"type": "registry:file",
"target": "agent/instrumentation.ts"
+ },
+ {
+ "path": "registry/instrumentation/braintrust-hook.ts",
+ "content": "import { braintrustEveHook } from \"braintrust\";\nimport { defineState } from \"eve/context\";\nimport { defineHook } from \"eve/hooks\";\n\nexport default defineHook(\n braintrustEveHook({\n defineState,\n metadata: {\n app: \"my-eve-agent\", // Replace with your app name\n },\n }) as Parameters<typeof defineHook>[0],\n);\n",
+ "type": "registry:file",
+ "target": "agent/hooks/braintrust.ts"
}
],
"envVars": {
"BRAINTRUST_API_KEY": ""
},
"type": "registry:item"
-}
+}
\ No newline at end of file
diff --git a/apps/docs/public/r/registry.json b/apps/docs/public/r/registry.json
index 001090cf4..027542244 100644
--- a/apps/docs/public/r/registry.json
+++ b/apps/docs/public/r/registry.json
@@ -1486,11 +1486,8 @@
"name": "instrumentation/braintrust",
"type": "registry:item",
"title": "Braintrust",
- "description": "Export AI SDK spans to Braintrust for tracing, evals, and monitoring.",
- "dependencies": [
- "@braintrust/otel",
- "@vercel/otel"
- ],
+ "description": "Trace eve agent sessions in Braintrust, including turns, steps, tool calls, and subagent interactions.",
+ "dependencies": ["braintrust"],
"envVars": {
"BRAINTRUST_API_KEY": ""
},
@@ -1499,6 +1496,11 @@
"path": "registry/instrumentation/braintrust.ts",
"type": "registry:file",
"target": "agent/instrumentation.ts"
+ },
+ {
+ "path": "registry/instrumentation/braintrust-hook.ts",
+ "type": "registry:file",
+ "target": "agent/hooks/braintrust.ts"
}
]
},
diff --git a/apps/docs/registry.json b/apps/docs/registry.json
index 001090cf4..027542244 100644
--- a/apps/docs/registry.json
+++ b/apps/docs/registry.json
@@ -1486,11 +1486,8 @@
"name": "instrumentation/braintrust",
"type": "registry:item",
"title": "Braintrust",
- "description": "Export AI SDK spans to Braintrust for tracing, evals, and monitoring.",
- "dependencies": [
- "@braintrust/otel",
- "@vercel/otel"
- ],
+ "description": "Trace eve agent sessions in Braintrust, including turns, steps, tool calls, and subagent interactions.",
+ "dependencies": ["braintrust"],
"envVars": {
"BRAINTRUST_API_KEY": ""
},
@@ -1499,6 +1496,11 @@
"path": "registry/instrumentation/braintrust.ts",
"type": "registry:file",
"target": "agent/instrumentation.ts"
+ },
+ {
+ "path": "registry/instrumentation/braintrust-hook.ts",
+ "type": "registry:file",
+ "target": "agent/hooks/braintrust.ts"
}
]
},
diff --git a/apps/docs/registry/instrumentation/braintrust-hook.ts b/apps/docs/registry/instrumentation/braintrust-hook.ts
new file mode 100644
index 000000000..715d4ae9f
--- /dev/null
+++ b/apps/docs/registry/instrumentation/braintrust-hook.ts
@@ -0,0 +1,12 @@
+import { braintrustEveHook } from "braintrust";
+import { defineState } from "eve/context";
+import { defineHook } from "eve/hooks";
+
+export default defineHook(
+ braintrustEveHook({
+ defineState,
+ metadata: {
+ app: "my-eve-agent", // Replace with your app name
+ },
+ }) as Parameters<typeof defineHook>[0],
+);
diff --git a/apps/docs/registry/instrumentation/braintrust.ts b/apps/docs/registry/instrumentation/braintrust.ts
index 27dd72e64..1d4180ab6 100644
--- a/apps/docs/registry/instrumentation/braintrust.ts
+++ b/apps/docs/registry/instrumentation/braintrust.ts
@@ -1,14 +1,15 @@
-import { BraintrustExporter } from "@braintrust/otel";
-import { registerOTel } from "@vercel/otel";
+import { braintrustEveInstrumentation, initLogger } from "braintrust";
+import { defineState } from "eve/context";
import { defineInstrumentation } from "eve/instrumentation";
-export default defineInstrumentation({
- setup: ({ agentName }) =>
- registerOTel({
- serviceName: agentName,
- traceExporter: new BraintrustExporter({
- parent: `project_name:${agentName}`,
- filterAISpans: true,
- }),
- }),
-});
+export default defineInstrumentation(
+ braintrustEveInstrumentation({
+ defineState,
+ setup: ({ agentName }) => {
+ initLogger({
+ projectName: agentName,
+ apiKey: process.env.BRAINTRUST_API_KEY,
+ });
+ },
+ }) as Parameters<typeof defineInstrumentation>[0],
+);
diff --git a/apps/docs/scripts/validate-channel-registry.ts b/apps/docs/scripts/validate-channel-registry.ts
index dd76b65fc..73db8eed5 100644
--- a/apps/docs/scripts/validate-channel-registry.ts
+++ b/apps/docs/scripts/validate-channel-registry.ts
@@ -12,7 +12,14 @@ interface RegistryItem {
name: string;
dependencies?: string[];
files?: RegistryFile[];
- meta?: { eve?: { channel?: string } };
+ meta?: {
+ eve?: {
+ setup?: {
+ command?: string;
+ args?: string[];
+ };
+ };
+ };
}
interface Registry {
@@ -85,9 +92,11 @@ for (const [index, item] of items.entries()) {
const registrySlug = expectedSlugs[index];
if (entry.slug === "slack" || entry.slug === "eve") {
- if (item.files !== undefined || item.meta?.eve?.channel !== registrySlug) {
+ const setup = item.meta?.eve?.setup;
+ const expectedArgs = ["integration", "setup", registrySlug];
+ if (setup?.command !== "eve" || JSON.stringify(setup.args) !== JSON.stringify(expectedArgs)) {
throw new Error(
- `Registry item "${item.name}" must delegate to eve channels add ${registrySlug}.`,
+ `Registry item "${item.name}" must delegate setup to eve integration setup ${registrySlug}.`,
);
}
continue;
diff --git a/apps/docs/scripts/validate-instrumentation-registry.ts b/apps/docs/scripts/validate-instrumentation-registry.ts
index 525a81688..960fefddc 100644
--- a/apps/docs/scripts/validate-instrumentation-registry.ts
+++ b/apps/docs/scripts/validate-instrumentation-registry.ts
@@ -47,15 +47,25 @@ if (JSON.stringify(actualSlugs) !== JSON.stringify(expectedSlugs)) {
for (const item of items) {
const slug = item.name.slice("instrumentation/".length);
const expectedPath = `registry/instrumentation/${slug}.ts`;
- const file = item.files?.[0];
- if (
- item.files?.length !== 1 ||
- file?.path !== expectedPath ||
- file.target !== "agent/instrumentation.ts"
- ) {
- throw new Error(
- `Registry item "${item.name}" must write ${expectedPath} to agent/instrumentation.ts.`,
- );
+ const expectedFiles: RegistryFile[] = [
+ {
+ path: expectedPath,
+ target: "agent/instrumentation.ts",
+ },
+ ...(slug === "braintrust"
+ ? [
+ {
+ path: "registry/instrumentation/braintrust-hook.ts",
+ target: "agent/hooks/braintrust.ts",
+ },
+ ]
+ : []),
+ ];
+ const actualFiles = item.files?.map(({ path, target }) => ({ path, target }));
+ if (JSON.stringify(actualFiles) !== JSON.stringify(expectedFiles)) {
+ throw new Error(`Registry item "${item.name}" does not contain the expected files.`);
+ }
+ for (const file of expectedFiles) {
+ await access(join(docsRoot, file.path));
}
- await access(join(docsRoot, expectedPath));
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ccd28989d..003b2c10c 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -264,9 +264,6 @@ importers:
'@beeper/chat-adapter-matrix':
specifier: 0.2.0
- '@braintrust/otel':
- specifier: ^0.3.0
- version: 0.3.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected](@aws-sdk/[email protected])([email protected])([email protected]))
'@browserbasehq/eve':
specifier: ^0.1.0
version: 0.1.0(eve@packages+eve)([email protected])
@@ -357,6 +354,9 @@ importers:
'@zernio/chat-sdk-adapter':
specifier: 0.5.0
+ braintrust:
+ specifier: 3.24.0
chat:
specifier: 4.34.0
@@ -1734,6 +1734,44 @@ packages:
'@braintree/[email protected]':
resolution: {integrity: sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA==}
+ '@braintrust/[email protected]':
+ resolution: {integrity: sha512-mY6VW/3VwcOQOGN8sYHS6F0xzHTFwgZcNlj7zlQttI6OXOCGt/bhonGIqd03QBhxmj0M31ymSS7TqSeCK/RYIQ==}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@braintrust/[email protected]':
+ resolution: {integrity: sha512-woyRyDv2DfCF8+von+3X9f1cddAbFnKLSfCbEkrBQVc0ZsAM60as8nehYv7DkafJF/67yKFx/sUraV65sukesQ==}
+ cpu: [x64]
+ os: [darwin]
+
+ '@braintrust/[email protected]':
+ resolution: {integrity: sha512-J9/7f3EIMKmFmSSQHQnAQLpUgB8YJENrOlkABjlTprBgsIYVgz7tSH7yg2P3ur+2Jem7PpGnrDxHGh/UjRfjsg==}
+ cpu: [arm64]
+ os: [linux]
+ libc: [glibc]
+
+ '@braintrust/[email protected]':
+ resolution: {integrity: sha512-KnLgENOoztBXcH+mLFJe4bYzi67kSOuELOQrm4Hler35GjgaBMI1fFLIUjKRPAmyT9VIhBMhy1ICfGEFLueMEQ==}
+ cpu: [x64]
+ os: [linux]
+ libc: [musl]
+
+ '@braintrust/[email protected]':
+ resolution: {integrity: sha512-HJFbUl3HYYY1Ivw8OYBeIMcIsU9r7+fC9BzLWB5FdH7881ToKee2wbbLZssMCxFbMVeUxzEo+SzGD/1Z9Gk9CA==}
+ cpu: [x64]
+ os: [linux]
+ libc: [glibc]
+
+ '@braintrust/[email protected]':
+ resolution: {integrity: sha512-+7PkdBAmiqEJsWteGHP/Zs62F75PkHPA8m/ej4TOz/mHGKulUU0bZibw91KRqIB7J7jGi5ItvCiqkiGaLKLnyw==}
+ cpu: [arm64]
+ os: [win32]
+
+ '@braintrust/[email protected]':
+ resolution: {integrity: sha512-Q0c+pVUGm82n/7N8QJ5s6j/G/Q0GFzqOaTipHjkmElLbAOOEcfRH/uljdtIPNBiEQcrzqirS0GmOgiFkeQ3Txg==}
+ cpu: [x64]
+ os: [win32]
+
'@braintrust/[email protected]':
resolution: {integrity: sha512-QTlvCleJ41MCleOt7BhwfYaIEVLrRZIp9xoSQcTjhdl91yLURyqieFNBA3FP3m1GxAxZmK6cSKgxW9q52iGTHA==}
peerDependencies:
@@ -9279,6 +9317,12 @@ packages:
peerDependencies:
zod: ^3.25.34 || ^4.0
+ resolution: {integrity: sha512-jF2XK1AImY2jI6uPxxU/KNGyYIIsr4PhaUD1IsbdggSNLIBkwzmluKvCzs1OUVy6LbpL4T2ekv8t+83UQ64hEg==}
+ hasBin: true
+ peerDependencies:
+ zod: ^3.25.34 || ^4.0
+
resolution: {integrity: sha512-FQBYNK15VMslhLHpA7+n+n1GOlF1kId2xcCg7/j95f24AOF6VDYMNH4mFxF7KuaTdv627faazpOAjFzMrfJOUw==}
engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7}
@@ -16832,6 +16876,27 @@ snapshots:
'@braintree/[email protected]': {}
+ '@braintrust/[email protected]':
+ optional: true
+
+ '@braintrust/[email protected]':
+ optional: true
+
+ '@braintrust/[email protected]':
+ optional: true
+
+ '@braintrust/[email protected]':
+ optional: true
+
+ '@braintrust/[email protected]':
+ optional: true
+
+ '@braintrust/[email protected]':
+ optional: true
+
+ '@braintrust/[email protected]':
+ optional: true
+
'@braintrust/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected](@aws-sdk/[email protected])([email protected])([email protected]))':
dependencies:
'@opentelemetry/api': 1.9.1
@@ -24701,6 +24766,51 @@ snapshots:
- '@aws-sdk/credential-provider-web-identity'
- supports-color
+ dependencies:
+ '@next/env': 14.2.35
+ '@vercel/functions': 1.6.0(@aws-sdk/[email protected])
+ acorn: 8.17.0
+ acorn-import-attributes: 1.9.5([email protected])
+ ajv: 8.20.0
+ argparse: 2.0.1
+ astring: 1.9.0
+ cjs-module-lexer: 2.2.0
+ cli-progress: 3.12.0
+ cli-table3: 0.6.5
+ cors: 2.8.6
+ dc-browser: 1.0.4
+ dotenv: 16.6.1
+ esbuild: 0.28.1
+ esquery: 1.7.0
+ eventsource-parser: 1.1.2
+ express: 5.2.1([email protected])
+ http-errors: 2.0.1
+ meriyah: 6.1.4
+ minimatch: 10.2.5
+ module-details-from-path: 1.0.4
+ mustache: 4.2.0
+ pluralize: 8.0.0
+ semifies: 1.0.0
+ simple-git: 3.36.0([email protected])
+ source-map: 0.7.6
+ termi-link: 1.1.0
+ unplugin: 2.3.11
+ uuid: 11.1.1
+ zod: 4.4.3
+ zod-to-json-schema: 3.25.2([email protected])
+ optionalDependencies:
+ '@braintrust/bt-darwin-arm64': 0.12.0
+ '@braintrust/bt-darwin-x64': 0.12.0
+ '@braintrust/bt-linux-arm64': 0.12.0
+ '@braintrust/bt-linux-x64': 0.12.0
+ '@braintrust/bt-linux-x64-musl': 0.12.0
+ '@braintrust/bt-win32-arm64': 0.12.0
+ '@braintrust/bt-win32-x64': 0.12.0
+ transitivePeerDependencies:
+ - '@aws-sdk/credential-provider-web-identity'
+ - supports-color
+
dependencies:
baseline-browser-mapping: 2.10.43
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 919fa997b..f5aaeafe4 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -7,6 +7,7 @@ packages:
allowBuilds:
# Registry templates import types only; this optional build is not needed.
"@google/genai": false
+ braintrust: false
"@mongodb-js/zstd": true
"@nestjs/core": true
"@parcel/watcher": false