The Open Framework for Building Agents
docs: reorganize observability guides (#3385)
Chad Hietaladed6583ec573
diff --git a/apps/docs/lib/geistdocs/llms-index.ts b/apps/docs/lib/geistdocs/llms-index.tsindex 22b76b79f..31cbc9095 100644--- a/apps/docs/lib/geistdocs/llms-index.ts+++ b/apps/docs/lib/geistdocs/llms-index.ts@@ -63,7 +63,7 @@ Documentation links below point directly to Markdown. Remove the \`.md\` suffix- [Deploy to Vercel](${EVE_ORIGIN}/docs/guides/deployment/vercel.md): Build and deploy with Vercel Workflow and Vercel Sandbox.- [Self-Hosting](${EVE_ORIGIN}/docs/guides/deployment/self-hosting.md): Run eve as a Node service or container.- [Authentication](${EVE_ORIGIN}/docs/guides/auth-and-route-protection.md): Secure an agent's HTTP routes and establish caller identity.-- [Observability](${EVE_ORIGIN}/docs/guides/instrumentation.md): Trace agents with OpenTelemetry and inspect workflow metadata.+- [Instrumentation](${EVE_ORIGIN}/docs/observability/instrumentation.md): Trace agents with OpenTelemetry and inspect workflow metadata.- [Terminal UI](${EVE_ORIGIN}/docs/guides/dev-tui.md): Work with a local or deployed agent from the interactive terminal UI.## Tutorialdiff --git a/apps/docs/lib/geistdocs/redirects.ts b/apps/docs/lib/geistdocs/redirects.tsindex b0d6f797d..caf9bb498 100644--- a/apps/docs/lib/geistdocs/redirects.ts+++ b/apps/docs/lib/geistdocs/redirects.ts@@ -68,6 +68,13 @@ export const docsRedirects: DocsRedirect[] = [...createDocsRedirects("/deployment/vercel", "/guides/deployment/vercel"),...createDocsRedirects("/deployment/self-hosting", "/guides/deployment/self-hosting"),...createDocsRedirects("/self-hosting", "/guides/deployment/self-hosting"),+ ...createDocsRedirects("/guides/instrumentation", "/observability/instrumentation"),+ ...createDocsRedirects(+ "/guides/instrumentation-providers",+ "/observability/instrumentation-providers",+ ),+ ...createDocsRedirects("/guides/instrumentation-otel", "/observability/otel"),+ ...createDocsRedirects("/observability/instrumentation-otel", "/observability/otel"),...createDocsRedirects("/evals", "/evals/overview"),...createDocsRedirects("/advanced/evals", "/evals/overview"),...createDocsRedirects("/getting-started/installation", "/getting-started"),diff --git a/apps/docs/lib/geistdocs/sitemap-transform.test.ts b/apps/docs/lib/geistdocs/sitemap-transform.test.tsindex bf577b73c..5842e531e 100644--- a/apps/docs/lib/geistdocs/sitemap-transform.test.ts+++ b/apps/docs/lib/geistdocs/sitemap-transform.test.ts@@ -23,6 +23,7 @@ describe("transformSitemapMarkdown", () => {it.each([["/docs/guides/hooks", "Guide", "How-to"],+ ["/docs/observability/instrumentation", "Guide", "How-to"],["/docs/protocols/acp", "Guide", "How-to"],["/docs/tutorial/first-agent", "Conceptual", "How-to"],["/docs/concepts/context-control", "Conceptual", "Conceptual"],diff --git a/apps/docs/lib/geistdocs/sitemap-transform.ts b/apps/docs/lib/geistdocs/sitemap-transform.tsindex 8fe7c9896..ab7f1697c 100644--- a/apps/docs/lib/geistdocs/sitemap-transform.ts+++ b/apps/docs/lib/geistdocs/sitemap-transform.ts@@ -16,6 +16,7 @@ const machineTypeFor = (url: string): string | undefined => {url === "/docs/getting-started" ||url === "/docs/install-integrations" ||url.startsWith("/docs/guides/") ||+ url.startsWith("/docs/observability/") ||url.startsWith("/docs/protocols/") ||url.startsWith("/docs/tutorial/")) {diff --git a/apps/docs/lib/integrations/data.ts b/apps/docs/lib/integrations/data.tsindex c4175b4e7..660b7955c 100644--- a/apps/docs/lib/integrations/data.ts+++ b/apps/docs/lib/integrations/data.ts@@ -2240,7 +2240,7 @@ type InstrumentationPresentation = ChannelPresentation;const instrumentationPresentations: Record<string, InstrumentationPresentation> = {braintrust: {logo: "braintrust",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "observability", "evals", "monitoring"],install: `Add the Braintrust integration from eve's registry:@@ -2284,11 +2284,11 @@ export default defineInstrumentation(}) as Parameters<typeof defineInstrumentation>[0],);\`\`\``,- configure: `Create an API key in the Braintrust dashboard and expose it as \`BRAINTRUST_API_KEY\`. Replace the hook's \`app\` metadata with your app name. Spans land in the Braintrust project named after your agent. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,+ configure: `Create an API key in the Braintrust dashboard and expose it as \`BRAINTRUST_API_KEY\`. Replace the hook's \`app\` metadata with your app name. Spans land in the Braintrust project named after your agent. See the [instrumentation guide](/docs/observability/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,},"posthog-instrumentation": {logo: "posthog",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "observability", "generations", "analytics"],install: `Add PostHog AI Observability from eve's registry:@@ -2333,11 +2333,11 @@ export default defineInstrumentation({},});\`\`\``,- configure: `Copy your project token and client API host from PostHog's project settings and expose them as \`POSTHOG_PROJECT_TOKEN\` and \`POSTHOG_HOST\`. Remove the \`events\` handler to capture generations anonymously. PostHog groups turns using \`eve.session.id\` and preserves eve's trace hierarchy. See [PostHog's eve installation guide](https://posthog.com/docs/ai-observability/installation/eve) for verification steps and the [instrumentation guide](/docs/guides/instrumentation) for input and output capture controls.`,+ configure: `Copy your project token and client API host from PostHog's project settings and expose them as \`POSTHOG_PROJECT_TOKEN\` and \`POSTHOG_HOST\`. Remove the \`events\` handler to capture generations anonymously. PostHog groups turns using \`eve.session.id\` and preserves eve's trace hierarchy. See [PostHog's eve installation guide](https://posthog.com/docs/ai-observability/installation/eve) for verification steps and the [instrumentation guide](/docs/observability/instrumentation) for input and output capture controls.`,},"sentry-instrumentation": {logo: "sentry",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "observability", "otlp", "errors"],install: `Add Sentry instrumentation from eve's registry. Sentry ingests OTLP directly, so no Sentry SDK is required:@@ -2365,11 +2365,11 @@ export default defineInstrumentation({}),});\`\`\``,- configure: `Copy the OTLP traces endpoint and public key from your Sentry project under **Settings → Client Keys (DSN)** and expose them as environment variables. Sentry's OTLP intake accepts traces only, and span events are dropped at ingestion. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,+ configure: `Copy the OTLP traces endpoint and public key from your Sentry project under **Settings → Client Keys (DSN)** and expose them as environment variables. Sentry's OTLP intake accepts traces only, and span events are dropped at ingestion. See the [instrumentation guide](/docs/observability/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,},"datadog-instrumentation": {logo: "datadog",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "observability", "apm", "otlp"],install: `Add Datadog instrumentation from eve's registry:@@ -2394,12 +2394,12 @@ export default defineInstrumentation({}),});\`\`\``,- configure: `Datadog's direct OTLP trace intake is site-specific (for example \`datadoghq.com\` vs \`datadoghq.eu\`) and currently in Preview; look up the endpoint for your site in Datadog's OTLP intake docs. For production, Datadog recommends routing through an OpenTelemetry Collector with the Datadog exporter instead. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,+ configure: `Datadog's direct OTLP trace intake is site-specific (for example \`datadoghq.com\` vs \`datadoghq.eu\`) and currently in Preview; look up the endpoint for your site in Datadog's OTLP intake docs. For production, Datadog recommends routing through an OpenTelemetry Collector with the Datadog exporter instead. See the [instrumentation guide](/docs/observability/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,relatedResources: [incidentResponseGuide],},"honeycomb-instrumentation": {logo: "honeycomb",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "observability", "queries", "otlp"],install: `Add Honeycomb instrumentation from eve's registry. Honeycomb ingests OTLP directly:@@ -2425,11 +2425,11 @@ export default defineInstrumentation({}),});\`\`\``,- configure: `Create an ingest key under your Honeycomb environment settings and expose it as \`HONEYCOMB_API_KEY\`. Spans arrive in a dataset named after your agent (the OTel service name). EU teams use \`https://api.eu1.honeycomb.io/v1/traces\`. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,+ configure: `Create an ingest key under your Honeycomb environment settings and expose it as \`HONEYCOMB_API_KEY\`. Spans arrive in a dataset named after your agent (the OTel service name). EU teams use \`https://api.eu1.honeycomb.io/v1/traces\`. See the [instrumentation guide](/docs/observability/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,},arize: {logo: "arize",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "llm observability", "evaluation", "otlp"],install: `Add Arize instrumentation from eve's registry. Arize AX ingests OTLP directly:@@ -2459,11 +2459,11 @@ export default defineInstrumentation({}),});\`\`\``,- configure: `Copy the space ID and API key from your Arize AX space settings and expose them as \`ARIZE_SPACE_ID\` and \`ARIZE_API_KEY\`. The \`openinference.project.name\` resource attribute routes spans to a project named after your agent. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,+ configure: `Copy the space ID and API key from your Arize AX space settings and expose them as \`ARIZE_SPACE_ID\` and \`ARIZE_API_KEY\`. The \`openinference.project.name\` resource attribute routes spans to a project named after your agent. See the [instrumentation guide](/docs/observability/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,},raindrop: {logo: "raindrop",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "observability", "ai issues", "otlp"],install: `Add Raindrop instrumentation from eve's registry. Raindrop ingests OTLP directly:@@ -2491,11 +2491,11 @@ export default defineInstrumentation({}),});\`\`\``,- configure: `Create a write key in the Raindrop dashboard and expose it as \`RAINDROP_WRITE_KEY\`. Raindrop's Vercel AI SDK integration picks up the AI SDK spans eve emits on every turn. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,+ configure: `Create a write key in the Raindrop dashboard and expose it as \`RAINDROP_WRITE_KEY\`. Raindrop's Vercel AI SDK integration picks up the AI SDK spans eve emits on every turn. See the [instrumentation guide](/docs/observability/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,},jaeger: {logo: "jaeger",- docsHref: "/docs/guides/instrumentation",+ docsHref: "/docs/observability/instrumentation",keywords: ["otel", "opentelemetry", "tracing", "observability", "local", "self-hosted"],install: `Add Jaeger instrumentation from eve's registry:@@ -2525,7 +2525,7 @@ export default defineInstrumentation({docker run --rm -p 16686:16686 -p 4318:4318 jaegertracing/jaeger:latest\`\`\`-Point the exporter at your collector's OTLP HTTP endpoint when self-hosting. See the [instrumentation guide](/docs/guides/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,+Point the exporter at your collector's OTLP HTTP endpoint when self-hosting. See the [instrumentation guide](/docs/observability/instrumentation) for the trace hierarchy and the \`recordInputs\`/\`recordOutputs\` controls.`,},};diff --git a/docs/agent-config.md b/docs/agent-config.mdindex d30e6e7db..5bc086a1d 100644--- a/docs/agent-config.md+++ b/docs/agent-config.md@@ -342,7 +342,7 @@ During `eve dev`, ordinary dependencies are bundled into each retained runtime g| Per-tool approval (HITL) | `agent/tools/*.ts`, [Tools](./tools) || Inbound auth & network policy | the channel layer, [Auth & route protection](./guides/auth-and-route-protection) || Sandbox / workspace | `agent/sandbox/`, [Sandbox](./sandbox) |-| Telemetry & debugging | `agent/instrumentation.ts`, [Instrumentation](./guides/instrumentation) |+| Telemetry & debugging | `agent/instrumentation.ts`, [Instrumentation](./observability/instrumentation) |## What to read nextdiff --git a/docs/channels/custom.mdx b/docs/channels/custom.mdxindex fcb5fe882..6d8bf7f8f 100644--- a/docs/channels/custom.mdx+++ b/docs/channels/custom.mdx@@ -312,7 +312,9 @@ The projection is re-evaluated whenever adapter state changes after channel even### Conversation audience-Use `audience(input)` to classify who can observe the conversation. The hook runs after route authentication and before the session is created:+Use `audience(input)` to assign an observability audience. The audience controls+whether instrumentation may capture conversation content. The hook runs after+route authentication and before the session is created:```tsimport { defineChannel, POST } from "eve/channels";@@ -342,8 +344,6 @@ audience({ auth }) {Classification is fixed when the session is created. Continuation turns from a different caller do not reclassify an existing session. When a parent dispatches a local subagent, the child inherits the parent's audience while the framework forwards the parent's custom channel metadata separately.-`metadata(state)` remains a channel-owned projection of custom observability fields. It no longer carries a reserved `audience` key. Channel extensions built before channel epoch 20 can still use that key; eve accepts it as a deprecated fallback and warns once, then authors should move classification to `audience(input)`.-## Continuation tokensEach channel operation accepts a channel-local token. The framework prepends the channel name, derived from the file stem under `agent/channels/`, before handing the token to the runtime.diff --git a/docs/channels/eve.mdx b/docs/channels/eve.mdxindex 9af0b28ee..2580e55a9 100644--- a/docs/channels/eve.mdx+++ b/docs/channels/eve.mdx@@ -159,7 +159,8 @@ For the full auth model and helper list, see [Auth & route protection](../guides## Audience-The eve channel classifies who can observe a session when that session is created:+The eve channel assigns an observability audience when it creates a session. The+audience controls whether instrumentation may capture session content:| Session creator | Default audience || ----------------------------------------- | ---------------- |@@ -169,32 +170,31 @@ The eve channel classifies who can observe a session when that session is createAnonymous HTTP surfaces are `unknown` by default because reachability does not establish that their content is safe to record. Authenticated `user`, `service`, and `runtime` sessions are private because they belong to one identified party. Other authenticated principal types also remain `unknown`: trace consumers record metadata but omit content by default in preview and production. Set an explicit `audience: "public"` only for intentionally public traffic.-Pass a constant or a function to override the default:+For a production endpoint whose conversations are all intentionally public,+set a constant audience:```ts title="agent/channels/eve.ts"-import { none, vercelOidc } from "eve/channels/auth";+import { vercelOidc } from "eve/channels/auth";import { eveChannel } from "eve/channels/eve";export default eveChannel({auth: [vercelOidc()],- audience: "private",+ audience: "public",});```-```ts title="agent/channels/eve.ts"-import { vercelOidc } from "eve/channels/auth";-import { eveChannel } from "eve/channels/eve";+`audience` controls observability content capture; it does not change who may+call the endpoint. `auth` still controls access. For a classification that+depends on channel state or application-specific verified data, use a [custom+channel audience](./custom#conversation-audience).-export default eveChannel({- auth: [vercelOidc(), none()],- audience({ caller, environment }) {- if (caller.type === "anonymous") return environment === "development" ? "public" : "unknown";- return "private";- },-});-```+You can also pass a function that receives `caller`, channel, run mode, and+deployment environment. Classification is fixed at session creation, so a later+turn from a different caller does not reclassify the session.-The function receives `caller`, channel, run mode, and deployment environment. `caller` is either `{ type: "anonymous" }` or a principal projection with `kind`, `authenticator`, and `attributes`. Classification is fixed at session creation; a continuation turn from a different caller does not reclassify the session. When another eve deployment forwards a request without an accepted forwarded trace assertion, the calling deployment's principal is used for classification, so the session remains `private`. A custom [trace policy](../guides/instrumentation-providers#control-inputs-and-outputs) can still admit or deny content independently.+For a remote agent, a trusted dispatch can forward the original audience and+its trace-content ceiling. See [Preserving trace+content](../guides/remote-agents#preserving-trace-content).## Customizationdiff --git a/docs/getting-started.mdx b/docs/getting-started.mdxindex 34d3717d1..c9129f2b3 100644--- a/docs/getting-started.mdx+++ b/docs/getting-started.mdx@@ -156,7 +156,7 @@ Prefer the nested layout because it keeps application files separate from the au### Debug file discovery-Run `eve info` when eve does not discover a file. It lists the discovered surface and diagnostics so you can check the authored slot and root-versus-subagent boundary. eve also writes inspectable artifacts under `.eve/`; see [Observability](./guides/instrumentation) and the [CLI](./reference/cli) reference.+Run `eve info` when eve does not discover a file. It lists the discovered surface and diagnostics so you can check the authored slot and root-versus-subagent boundary. eve also writes inspectable artifacts under `.eve/`; see [Instrumentation](./observability/instrumentation) and the [CLI](./reference/cli) reference.## Install manuallydiff --git a/docs/guides/deployment/self-hosting.md b/docs/guides/deployment/self-hosting.mdindex fad0f11bb..f54728fc9 100644--- a/docs/guides/deployment/self-hosting.md+++ b/docs/guides/deployment/self-hosting.md@@ -84,5 +84,5 @@ eve dev https://your_agent.example.comUse these guides to secure and observe the deployed agent:- [Authentication](../auth-and-route-protection): configure the host’s route policy-- [Observability](../instrumentation): export traces and diagnose runtime failures+- [Instrumentation](../../observability/instrumentation): export traces and diagnose runtime failures- [Sandbox](../../sandbox): select and secure a sandbox backenddiff --git a/docs/guides/deployment/vercel.mdx b/docs/guides/deployment/vercel.mdxindex 685dc586d..6396f2be0 100644--- a/docs/guides/deployment/vercel.mdx+++ b/docs/guides/deployment/vercel.mdx@@ -158,12 +158,12 @@ Set `VERCEL_AUTOMATION_BYPASS_SECRET` locally before connecting if the deploymenVercel detects eve and can add an **Agent Runs** tab under the project’s **Observability** view. Use it to browse sessions and inspect each conversation trace.-The Agent Runs tab requires enablement for your Vercel team. Contact your Vercel representative if the tab doesn’t appear. For third-party tracing backends, configure [OpenTelemetry instrumentation](../instrumentation).+The Agent Runs tab requires enablement for your Vercel team. Contact your Vercel representative if the tab doesn’t appear. For third-party tracing backends, configure [OpenTelemetry instrumentation](../../observability/instrumentation).## Continue configuring productionUse these guides to secure and observe the deployed agent:- [Authentication](../auth-and-route-protection): configure who can call the deployed agent-- [Observability](../instrumentation): export traces and diagnose runtime failures+- [Instrumentation](../../observability/instrumentation): export traces and diagnose runtime failures- [Sandbox](../../sandbox): configure resources, isolation, and network accessdiff --git a/docs/guides/dev-tui.md b/docs/guides/dev-tui.mdindex d90168713..004f2864d 100644--- a/docs/guides/dev-tui.md+++ b/docs/guides/dev-tui.md@@ -77,7 +77,7 @@ By default, the UI shows `stderr` logs. Use `/loglevel <all|stderr|sandbox|none>Every `eve dev` process writes diagnostic logs to `.eve/logs/`, regardless of the display mode. Read them with [`eve logs`](../reference/cli#eve-logs).-Use `/traces` to inspect traces recorded during local development. See [Instrumentation](instrumentation#local-traces) for trace capture and retention settings.+Use `/traces` to inspect traces recorded during local development. See [Instrumentation](../observability/instrumentation#local-traces) for trace capture and retention settings.## Display options@@ -108,6 +108,6 @@ For a Vercel deployment that needs authentication, run `/vc:login` and follow th## What to read next-- [Instrumentation](./instrumentation): traces, OpenTelemetry, and diagnostics.+- [Instrumentation](../observability/instrumentation): traces, OpenTelemetry, and diagnostics.- [CLI](../reference/cli): commands and flags.- [Agent Client Protocol (ACP)](../protocols/acp): drive the same agent from ACP clients such as Zed instead of the TUI.diff --git a/docs/guides/hooks.md b/docs/guides/hooks.mdindex a4523477a..48d780ca3 100644--- a/docs/guides/hooks.md+++ b/docs/guides/hooks.md@@ -50,7 +50,7 @@ export default githubChannel({A GitHub channel event handler cannot fire for a Slack-owned session, so platform-specific side effects do not depend on an early-return guard. On a built-in channel, an authored handler replaces that channel's default handler for the same event key. Check the channel page before overriding events that deliver replies, progress, errors, or human-input prompts.-Use `ctx.channel.kind` inside a global hook only when the operation is otherwise agent-wide and conditional handling is intentional. For typed channel metadata in dynamic resolvers or instrumentation, import the channel definition and narrow with `isChannel`; see [Instrumentation](./instrumentation#runtime-context).+Use `ctx.channel.kind` inside a global hook only when the operation is otherwise agent-wide and conditional handling is intentional. For typed channel metadata in dynamic resolvers or instrumentation, import the channel definition and narrow with `isChannel`; see [Add runtime context](../observability/instrumentation#add-runtime-context).## Hook structure and contextdiff --git a/docs/guides/instrumentation-providers.md b/docs/guides/instrumentation-providers.mddeleted file mode 100644index c8fd46f9d..000000000--- a/docs/guides/instrumentation-providers.md+++ /dev/null@@ -1,249 +0,0 @@-----title: "Instrumentation Providers"-description: "Configure the experimental instrumentation provider layout, control captured inputs and outputs, and redact OpenTelemetry spans before export."------Instrumentation providers split observability into files under `agent/instrumentation/`. Each file can handle eve lifecycle events or add an OpenTelemetry destination without owning the rest of the telemetry pipeline.--**This API is experimental and may change without a deprecation period.** Enable it explicitly in `agent.ts`:--```ts title="agent/agent.ts"-import { defineAgent } from "eve";--export default defineAgent({- model: "anthropic/claude-sonnet-5",- experimental: {- instrumentationProviders: true,- },-});-```--The provider directory replaces `agent/instrumentation.ts`; the two layouts cannot be used together. See [Observability](./instrumentation) for the default `instrumentation.ts` API.--## Add a provider--The filename identifies the provider slot. Each file must default-export `defineInstrumentation(...)`, an OpenTelemetry integration, or `disableInstrumentation()`.--```text-agent/instrumentation/- audit.ts lifecycle event provider- braintrust.ts OpenTelemetry destination- otel.ts process-wide OpenTelemetry settings-```--This provider records action timing and identity without receiving tool arguments or results:--```ts title="agent/instrumentation/audit.ts"-import { defineInstrumentation } from "eve/instrumentation";--export default defineInstrumentation({- events: {- "action.started": (event, ctx) => {- ctx.state.set({ name: event.name, startedAt: Date.now() });- },- "action.completed": (event, ctx) => {- const started = ctx.state.get() as { name: string; startedAt: number } | undefined;- if (started === undefined) return;-- console.log({- action: started.name,- durationMs: Date.now() - started.startedAt,- outcome: event.outcome,- });- },- },-});-```--`ctx.state` is JSON storage scoped to this provider and operation. It survives durable suspension and is released after the terminal event.--## Control inputs and outputs--Each provider has an independent `tracePolicy`. It decides whether the provider receives events and whether those events include input or output content.--```ts title="agent/instrumentation/audit.ts"-import { defineInstrumentation } from "eve/instrumentation";--export default defineInstrumentation({- tracePolicy: ({ audience, environment }) => ({- emit: true,- recordInputs: audience === "public" || environment === "development",- recordOutputs: audience === "public" || environment === "development",- }),- events: {- "model.call.started": (event) => {- console.log("input", event.input);- },- "model.call.completed": (event) => {- console.log("output", event.content);- },- },-});-```--The policy receives `agentName`, `channel`, `audience`, `mode`, `environment`, and `principalType`. `audience` is `"public"`, `"private"`, or `"unknown"`. `environment` is `"development"`, `"preview"`, or `"production"`.--`principalType` is the principal used for classification. Normal channels report their route or session principal; forwarded eve sessions report the calling deployment's principal. Sessions that predate the durable conversation key report their session initiator.--The default policy emits metadata for every audience and includes inputs and outputs for `public` conversations and for any audience in a development environment. Preview behaves like production: an `unknown` conversation records metadata only. Return an explicit decision to change that behavior:--| Decision | Result |-| --------------------------------------------- | ------------------------------------------------------------ |-| `false` or `{ emit: false }` | Do not invoke this provider for the trace. |-| `true` | Emit the trace with the default audience-aware content rule. |-| `{ emit: true, recordInputs, recordOutputs }` | Emit the trace with the selected content directions. |--Inputs include model prompts, tool arguments, channel input, and user responses. Outputs include model responses, tool results, requests for user input, provider metadata, and error details. Content fields are optional on event types because the active policy may remove them.--A provider's policy does not change another provider or the OpenTelemetry pipeline. A policy that throws fails closed for that provider.--## Redact fields in a custom provider--Lifecycle events are immutable snapshots. Copy the fields you need into a destination-specific payload and redact that copy before sending it:--```ts title="agent/instrumentation/audit.ts"-import { defineInstrumentation } from "eve/instrumentation";--export default defineInstrumentation({- tracePolicy: () => ({- emit: true,- recordInputs: true,- recordOutputs: false,- }),- events: {- "action.started": async (event) => {- await sendAuditRecord({- id: event.idempotencyKey,- input: redactApiKey(event.input),- kind: event.kind,- name: event.name,- });- },- },-});--function redactApiKey(value: unknown): unknown {- if (typeof value !== "object" || value === null || Array.isArray(value)) return value;-- const record = value as Record<string, unknown>;- return "apiKey" in record ? { ...record, apiKey: "[redacted]" } : record;-}--async function sendAuditRecord(record: unknown): Promise<void> {- // Send the sanitized record to your provider.- void record;-}-```--Prefer `recordInputs: false` or `recordOutputs: false` when the destination does not need an entire content direction. Use field-level redaction only when the destination needs part of that content.--## Add an OpenTelemetry destination--OpenTelemetry configuration has two parts:--- `otel()` declares process-wide settings such as the resource, sampler, propagators, and trace capture policy. Declare it at most once.-- `otelIntegration()` adds one destination. Declare one file per exporter or processor chain.--Most agents only need a destination:--```ts title="agent/instrumentation/braintrust.ts"-import { BraintrustExporter } from "@braintrust/otel";-import { otelIntegration } from "eve/instrumentation/otel";--export default otelIntegration({- traceExporter: new BraintrustExporter({ filterAISpans: true }),-});-```--Custom destinations receive the schema v4 agent trace contract. Follow [Query exported traces](./instrumentation#query-exported-traces) to find activations and sessions across traces.--Add `agent/instrumentation/otel.ts` when you need process-wide settings or want to control which content eve writes to OpenTelemetry spans:--```ts title="agent/instrumentation/otel.ts"-import { otel } from "eve/instrumentation/otel";--export default otel({- resource: { "deployment.environment": process.env.VERCEL_ENV ?? "development" },- tracePolicy: ({ audience, environment }) => ({- emit: true,- recordInputs: audience === "public" || environment === "development",- recordOutputs: audience === "public" || environment === "development",- }),-});-```--The OpenTelemetry `tracePolicy` is a capture ceiling shared by its destinations. A destination cannot restore content excluded by this policy. It does not affect lifecycle event providers created with `defineInstrumentation()`.--## Redact managed OpenTelemetry destinations--`localTraces()` and `agentRuns()` accept an `exportPolicy`. Use the built-in redactors to remove known input or output attributes before that destination's processors receive a span:--```ts title="agent/instrumentation/agent-runs.ts"-import {- agentRuns,- composeSpanExportPolicies,- redactSpanInputs,- redactSpanOutputs,-} from "eve/instrumentation/otel";--export default agentRuns({- exportPolicy: composeSpanExportPolicies(- redactSpanInputs(({ audience }) => audience !== "public"),- redactSpanOutputs(({ audience }) => audience !== "public"),- {- attribute: ({ key }) =>- key === "user.email" ? { action: "replace", value: "[redacted]" } : { action: "keep" },- },- ),-});-```--`redactSpanInputs()` removes known prompts, instructions, documents, and tool arguments. `redactSpanOutputs()` removes known responses, reasoning, tool results, exception details, event attributes, and status messages. Their optional predicate receives the span name, IDs, attributes, and channel audience.--An export policy can also remove spans or attributes:--```ts-{- span: ({ name }) => name !== "internal.cache.refresh",- attribute: ({ key }) =>- key === "customer.id" ? { action: "drop" } : { action: "keep" },-}-```--Policies run in declaration order. A later policy sees the filtered span produced by earlier policies. Redaction and filtering apply only to that destination and do not mutate spans shared with other destinations.--`recordInputs` and `recordOutputs` remain accepted by `localTraces()` and `agentRuns()` for compatibility, but are deprecated. Use `redactSpanInputs()` and `redactSpanOutputs()` in `exportPolicy` instead.--`otelIntegration()` does not accept `exportPolicy`. Use the process-wide `otel({ tracePolicy })` to limit capture for all custom OpenTelemetry destinations, or supply a destination-specific `SpanProcessor` that filters before its exporter.--## Built-in slots--The provider layout adds two environment-specific defaults:--- `local` records local traces during `eve dev`.-- `agent-runs` exports to Vercel Agent Runs in preview and production deployments.--Omitting these files preserves the defaults. Reconfigure a slot by exporting `localTraces()` or `agentRuns()` from the matching file. Disable one explicitly:--```ts title="agent/instrumentation/local.ts"-import { disableInstrumentation } from "eve/instrumentation";--export default disableInstrumentation();-```--## Lifecycle events--Providers can handle session, channel delivery, turn, model attempt, model call, action, input request, and tool call events. Start and terminal events share an `idempotencyKey`, which can serve as a destination row ID.--An ordinary tool emits both `action.*` and `tool.call.*` events. Use `action.*` for eve's durable dispatch lifecycle, including tools, skills, subagents, and remote agents. Use `tool.call.*` only when you need the AI SDK's in-process tool execution boundary.--Handlers for different providers run concurrently and are failure-isolated. Do not depend on provider execution order. Use `flush` to drain buffered records and `shutdown` to release resources.--The [agent trace contract](./instrumentation#agent-trace-contract) defines span-names, identities, and usage counters shared by the exporters and local viewers.--## What to read next--- [Observability](./instrumentation): the default `instrumentation.ts` API and trace hierarchy-- [Local development](./dev-tui): inspect local traces in the TUI-- [Hooks](./hooks): react to runtime events outside the instrumentation provider APIdiff --git a/docs/guides/instrumentation.md b/docs/guides/instrumentation.mddeleted file mode 100644index 8be8c3e80..000000000--- a/docs/guides/instrumentation.md+++ /dev/null@@ -1,472 +0,0 @@-----title: "Observability"-description: "Trace an agent with OpenTelemetry in instrumentation.ts, read the workflow run tags eve emits, and debug discovery with eve info and the common-failures table."------`instrumentation.ts` is where you configure how an eve agent is observed. The framework auto-discovers `agent/instrumentation.ts` and runs it at server startup before any agent code. Its presence implicitly enables telemetry, so there is no separate `isEnabled` toggle.--If you intend to export telemetry, review the exporter destination, data categories, and required legal approvals before enabling telemetry.--**The instrumentation provider API is experimental and off by default.** Enable `experimental.instrumentationProviders` to use its one-file-per-provider layout, directional content capture, and per-destination redaction. See [Instrumentation Providers](./instrumentation-providers) for setup and current limitations.--## Three observability surfaces--eve observes an agent through three distinct surfaces. They do not all live in this file, and they write to different places:--| Surface | Configured in `instrumentation.ts`? | What it is |-| -------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |-| **Workflow run tags** (`$eve.*`) | No (automatic) | Framework-owned attributes on each Vercel Workflow run. Let dashboards stitch session, turn, and subagent runs into a tree and surface model and token usage. |-| **OpenTelemetry export** | Local: automatic. Authored: `setup` and capture settings | Where agent and AI spans are exported and what they record. |-| **Runtime context events** | Yes: `events["step.started"]` | Per-model-call values written into the AI SDK's runtime context, which the AI SDK carries onto its spans. |--The two configurable surfaces send AI SDK spans to your OpenTelemetry backend. Workflow run tags are a separate system, queryable in the Workflow dashboard rather than on your OTel spans. The sections below cover what you configure here; [Workflow run tags](#workflow-run-tags) documents what eve emits on its own.--## Define instrumentation--```ts title="agent/instrumentation.ts"-import { BraintrustExporter } from "@braintrust/otel";-import { defineInstrumentation } from "eve/instrumentation";-import { registerOTel } from "@vercel/otel";--export default defineInstrumentation({- setup: ({ agentName }) =>- registerOTel({- serviceName: agentName,- traceExporter: new BraintrustExporter({- parent: `project_name:${agentName}`,- filterAISpans: true,- }),- }),-});-```--Export the result of `defineInstrumentation` as the default export.--## OpenTelemetry--Use the `setup` callback to register your OTel provider (for example `registerOTel` from `@vercel/otel`). The framework invokes it at server startup with the resolved agent name. `context.agentName` is resolved at compile time from your project (the package's `name`, falling back to the app directory name), so you never hard-code a service name.--Any OTel-compatible backend works (Braintrust, PostHog, Sentry, Raindrop, Arize, Honeycomb, Datadog, Jaeger). Install the exporter package you need and configure it in the callback. The [PostHog AI Observability integration](/integrations/posthog-instrumentation) provides a ready-to-install exporter and optional user identification. The [Sentry integration](/integrations/sentry-instrumentation) provides a ready-to-install OTLP exporter that sends traces to Sentry without a Sentry SDK.--Three more fields control what eve and the AI SDK record inside those spans (see the AI SDK's [telemetry reference](https://ai-sdk.dev/docs/ai-sdk-core/telemetry)):--- `recordInputs` records full message history on each step span and recalled records on memory spans. It defaults to `false`; set it to `true` to include input content.-- `recordOutputs` records model outputs. It defaults to `false`; set it to `true` to include output content.-- `functionId` overrides the function name on spans (defaults to the agent name).--eve records metadata without model, tool, or memory-record content by default. Enable either content category only after reviewing the exporter and its data-retention path.--In the provider layout, eve stamps each span with `gen_ai.conversation.id`, which stays fixed across local and remote activations, including independent remote root workflows. Query this attribute to find the conversation's exported, retained traces; it does not grant access or control trace parenting. On Vercel, `vercel.session_id` identifies the verified root session, while `agent.run.id` identifies the current agent run. See [Query exported traces](#query-exported-traces) for the cross-backend workflow.--You are responsible for ensuring any observability or eval provider is approved for the data exported to it.--The third configurable surface, [runtime context events](#runtime-context), attaches per-model-call values to these spans.--Channels classify their conversation with an `audience`: `public`, `private`, or `unknown`. The eve channel classifies anonymous callers as unknown and authenticated `user`, `service`, or `runtime` callers as private. Slack public-channel handoffs and Chat SDK workspace-visible threads are public; direct and private conversations are private; platform surfaces without enough visibility evidence remain unknown. Inbound Slack webhooks remain `unknown`; proactive Slack `receive` / `ctx.send` handoffs also stay `unknown` unless the caller passes `audience` on the target, for example when a webhook or schedule already knows the destination channel is public.--## Channel delivery traces--Instrumentation providers receive `channel.delivery.started` followed by-`channel.delivery.completed`, `channel.delivery.cancelled`, or-`channel.delivery.failed` for every inbound channel operation. The lifecycle-covers durable processing through the terminal state of the resulting turn, not-messages an adapter sends back to Slack, Telegram, Twilio, or another platform.-Adjacent queued messages with matching authenticated identities and authorization-attributes can share a turn while retaining separate lifecycle pairs. An adapter-can also consume a delivery without starting a turn.--Each operation has a framework-owned `deliveryId` distinct from its optional-platform request ID. Metadata-only providers receive identity, channel, session,-and outcome fields. Content providers additionally receive only eve's known-message, context, input-response, and output-schema fields; adapter-specific-payload fields are never projected.--For a single delivery that starts a turn, the built-in OpenTelemetry provider records the channel kind, channel name, delivery ID, optional request ID, and captured input on that turn's `invoke_agent` activation. The activation remains a root in its own trace and links to the active upstream request or function span with `eve.link.type=channel.request`. Set `traceChannelRequests: true` to create an eve-owned HTTP server span as that link target; when the option is false, an already-active upstream span remains the target.--Deliveries that do not map one-to-one to an activation still produce instrumentation lifecycle events, but the built-in OpenTelemetry provider does not emit a separate delivery span.--## Callback delivery errors--Failed outbound session and task callback attempts emit an error-level-`[eve:execution.session-callback] callback delivery failed` runtime log without-requiring an instrumentation provider. Filter by `statusCode` for HTTP failures-or `failure` (`http`, `transport`, or `timeout`). The log includes the callback-origin, token-redacted route, payload kind, and available call, task, and child-session identifiers. Payload content, credentials, and callback tokens are-excluded. Each retry can emit a separate error; logging does not change Workflow-retry behavior. Best-effort activity delivery keeps its single-`[eve:execution.activity-submit] activity sink request failed` warning and does-not mark the active span as failed.--## Exported span names and outcomes--The built-in OpenTelemetry provider preserves each span's OTel name and adds-`operation.name` and `resource.name` for Datadog's operation/resource mapping:--| OTel span name | `operation.name` | `resource.name` |-| ----------------------------------------------------------------------- | ----------------- | ------------------------ |-| `invoke_agent weather` | `invoke_agent` | `invoke_agent weather` |-| `invoke_workflow deploy` | `invoke_workflow` | `invoke_workflow deploy` |-| `execute_tool search` | `execute_tool` | `execute_tool search` |-| `chat <model>` | `chat` | `chat <model>` |-| `search_memory`, `upsert_memory` | Same as span name | Same as span name |-| `agent.step`, `agent.action`, `agent.approval`, `agent.channel.request` | Same as span name | Same as span name |--These rows apply to eve-owned spans in local tracing and the-[instrumentation provider layout](./instrumentation-providers). They do not-rename Workflow, AI SDK, or other third-party spans, or change trace IDs,-parenting, or session grouping. Legacy `instrumentation.ts` setup still owns-its exporter configuration and [authored trace hierarchy](#authored-trace-hierarchy).--Workflow action spans enter the sampler as `agent.action`, with-`gen_ai.operation.name` and `gen_ai.workflow.name` already present. After-sampling, eve changes the exported span name to `invoke_workflow <tool>`.-Existing name-based samplers therefore keep their `agent.action` behavior;-attribute-based samplers can select GenAI workflow operations explicitly.-Review custom attribute rules when upgrading because the new workflow-attributes may change their decisions.--In Datadog APM, `operation_name:invoke_agent resource_name:"invoke_agent weather"`-selects named agent invocations. Operation-based dashboards and monitors may-need to replace inferred names such as `otel.span` with these explicit names.-Keep session IDs, turn IDs, and message content in attributes, not operation or-resource names.--If an eve-owned span still appears as `otel.span`, inspect its exported OTel-name, `operation.name`, and `resource.name` before and after any drain or-Collector transforms. Datadog's [operation-name mapping-guide](https://docs.datadoghq.com/opentelemetry/migrate/migrate_operation_names/)-describes the explicit override. Exported fields must survive the ingestion-path; a local export does not prove that a hosted backend retained them.--Agent invocation spans carry `agent.turn.outcome=completed|failed|cancelled`-when the turn has a terminal outcome. A failed invocation also has OTel error-status; cancellation is not an error. The scalar outcome survives backends-that discard span events, including Sentry's [direct OTLP-intake](https://docs.sentry.io/concepts/otlp/direct/traces/). These attributes-remain available when model and tool content is redacted.--## Memory spans--eve records [OpenTelemetry GenAI memory spans](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#memory) for provider calls. Recalls at `turn.started` and `compaction.completed` use `search_memory`; capture handlers use `upsert_memory`. They are client spans, following the GenAI memory convention for a call to a memory system.--Every memory span includes `gen_ai.operation.name` and `gen_ai.memory.store.id`. The store ID is eve's opaque `memory.scope.key`, which identifies the resolved scope without exposing the namespace or scope values. Recall spans set `gen_ai.memory.record.count` to their result count.--`gen_ai.memory.records` contains recalled record content only when `recordInputs` is enabled. eve classifies recalled records as input content because they become part of the model context. eve does not set `gen_ai.memory.query.text` because a memory provider receives structured conversation messages, not a standalone search-query string. The `agent.memory.slot` and `agent.memory.phase` attributes identify the eve slot and lifecycle boundary without adding either to the span name.--Messages in `gen_ai.input.messages` include a `kind` for user-role entries.-New human input uses `user`; framework-authored input uses namespaced kinds-such as `context.instruction` and `execution.background_task`. When resuming-history saved before eve 0.54, entries without a kind use `legacy.unknown`:-the old format did not reliably distinguish human input from framework input.-Their content and metadata remain unchanged, and already classified entries-retain their kind. Unknown legacy entries are not replayed as the latest human-request after compaction.--## Agent trace contract--The provider layout and zero-config local tracing emit the following spans.-The legacy `instrumentation.ts` layout still uses its authored OTel setup.--| Span | Meaning |-| ------------------------ | -------------------------------------------------------- |-| `invoke_agent <agent>` | One agent activation in its own trace |-| `agent.step` | One model attempt |-| `chat <model>` | One model call beneath its step |-| `invoke_workflow <tool>` | One workflow tool run that coordinated a nested agent |-| `agent.action` | Durable action lifecycle, including dispatch and waiting |-| `execute_tool <tool>` | In-process tool execution beneath its action |-| `agent.approval` | Approval waiting beneath its action |-| `search_memory` | Recall memory records before a turn or after compaction |-| `upsert_memory` | Automatic memory capture |-| `agent.channel.request` | Optional HTTP request span in the provider layout |--For background tools and subagents, the AI SDK's `execute_tool` span ends when-the model receives the task receipt. The enclosing `agent.action` span remains-open until the background task completes, fails, or is cancelled. When a-workflow tool coordinates a nested `ctx.agent()` call, eve exports that span as-`invoke_workflow` instead. Its duration, outcome, and usage describe the-background task rather than the receipt.-Background tool results follow the output-content policy. Recorded task failure-details become bounded exception and status messages; redacted spans retain only-the failure outcome, error status, and error type. The initiating turn can-finish or be cancelled while the action remains open. Ending the session closes-an action whose task never reported a terminal result.--Schema v4 removes the session-long `agent.session` root. Every run-associated-eve span carries `agent.trace.schema.version=4`, `agent.run.id`, and-`gen_ai.conversation.id`; the pre-session `agent.channel.request` span is the-exception. Vercel deployments additionally carry-`vercel.session_id`, which stays fixed to the root session across local and-trusted remote subagents. Remote lineage is accepted only when the receiver's-`trustedForwarders` predicate approves the authenticated caller. Child-activation roots carry `agent.parent_run.id` and `agent.parent_call.id`.-Activation roots also carry `agent.run.type`, `agent.channel.audience`, and-directional `agent.trace.content.input` and `agent.trace.content.output` policy-results. Root-session activations carry their initiating `agent.channel.kind`-and `agent.session.origin` (`schedule` or `channel`), plus `agent.schedule.id`-when a schedule started the session and `agent.session.title` when input-capture permits. The title repeats on every turn activation for that session-and always describes the initial session input, not a later turn. A turn with-an actual channel delivery carries that delivery's channel kind, name, and ID.-Delivery-less subagent activations omit the root-only channel kind, origin,-schedule, and title attributes; their delegation is described by-`agent.run.type` and the parent attributes. `agent.run.type` is `session` or-`subagent`; each exported content-policy boolean reflects both the resolved-trace capture decision and that destination's redaction policy.-Only activations use the `invoke_agent` operation. A workflow tool invocation-that coordinates at least one nested agent uses `invoke_workflow`, with its-path-derived tool name in `gen_ai.workflow.name`. Durable workflow tools without-agent operations, including `sleep`, remain `agent.action` spans. Dispatch-lifecycle spans use `agent.action` with-`agent.invocation.role=caller`; the built-in `agent` tool executes under-`execute_tool agent`. Turn IDs such as `turn_0` are local to a session, so-correlate turns by session ID and turn ID together.--A conversation remains durable state; `gen_ai.conversation.id` joins the activations that operate on it.--Each activation owns a fresh trace identity, including local and remote-subagents. The first child activation links to its caller with-`eve.link.type=agent.dispatch`; remote dispatch preserves that caller separately-in W3C `tracestate` while HTTP intermediaries update `traceparent`. Older peers-fall back to the incoming transport context. Later turns do not reuse the-original caller link.-Conversation baggage is independent of execution lineage and trace-policy-ceilings, which remain in force across the boundary.--Channel delivery does not allocate or replace the activation, and a later turn-receives a new trace. Samplers-see the activation name and attributes after its session coordinates are-available. Sampling must be deterministic for the same trace and operation-because durable reconstruction can evaluate it again.--Use `agent.turn.outcome` to distinguish completed, failed, and cancelled activations. A failed action can be handled by the agent without failing its activation; see [Exported span names and outcomes](#exported-span-names-and-outcomes).--Activation spans retain `gen_ai.usage.input_tokens` and-`gen_ai.usage.output_tokens` alongside `agent.usage.*` totals for that activation.-Model spans carry `gen_ai.usage.*`; step and dispatch spans report-`agent.usage.*`. Activation totals summarize that activation's own model calls, while caller spans can summarize delegated usage. Do not sum usage across these levels.--Nested dispatch spans are materialized when the child settles and handed to-span processors; successful materialization removes their completed records-from durable state. Settlement does not drain exporters or authored providers.-Materialization failures are logged without failing settlement, and exporter-draining follows the runtime's normal flush lifecycle.--Cancelled or abandoned dispatches retain-their outcome without error status. Error messages and stacks on eve spans count as-output content, including errors reconstructed after a worker replacement.-Metadata-only capture retains failure status without those details. Error logging-outside eve's instrumented execution retains its existing exception content.--Agent Runs activation metadata includes bounded principal summaries:--- `agent.principal.current.type` and `agent.principal.current.id` describe the current caller.-- `agent.principal.initiator.type` and `agent.principal.initiator.id` describe the authenticated principal that created the root session. The initiator remains fixed when later turns have a different caller.--Types are limited to `user`, `service`, `runtime`, `app`, `anonymous`, `local-dev`, `unknown`, `none`, and `other`. Types are emitted for every audience when a principal is present. An absent type means no authentication context was set; `none` means an explicitly null principal and has no ID. The auth layer's `unknown` type stays `unknown`; unrecognized authored types become `other`.--Principal IDs require content-visible capture and a resolved trace decision that allows both `recordInputs` and `recordOutputs`. Public turns and every audience in a development environment can include IDs; private and unknown turns omit them in preview and production. A user-configured trace policy or forwarded content ceiling that denies either direction omits both principal IDs before turn state is stored or sampled. Empty IDs and IDs larger than 1 KiB of UTF-8 data are omitted, not truncated; authentication records are unchanged. eve does not copy other authentication fields, such as claims, email attributes, issuers, or subjects, into these summaries.--## Query exported traces--Use a trace to investigate one activation's work. Use session and conversation attributes to find the other activations. Exporting to an OpenTelemetry backend does not turn a conversation into one continuous waterfall.--This workflow applies to schema v4 from the [instrumentation provider layout](./instrumentation-providers). Configure third-party exporters through `otelIntegration()` so they receive the same framework spans as local tracing and Agent Runs. The legacy `instrumentation.ts` setup emits the [authored trace hierarchy](#authored-trace-hierarchy), not this contract.--### Find an activation or conversation--| Investigation | Filter |-| ------------------------- | ----------------------------------------------------------------------- |-| Agent activations | `agent.trace.schema.version=4` and `gen_ai.operation.name=invoke_agent` |-| One logical conversation | `gen_ai.conversation.id=<conversation ID>` |-| One turn | Both `gen_ai.conversation.id` and `agent.turn.id` |-| One Vercel root session | `vercel.session_id=<root session ID>` |-| One agent run | `agent.run.id=<run ID>` |-| Root agent runs | `agent.run.type=session` |-| Subagent runs | `agent.run.type=subagent` |-| Scheduled root sessions | `agent.session.origin=schedule` |-| One recorded channel kind | `agent.channel.kind=<channel kind>` |-| Exported input content | `agent.trace.content.input=true` |-| Exported output content | `agent.trace.content.output=true` |-| Delegated dispatches | `agent.invocation.role=caller` |--Start with an activation filter for latency, failure, or turn-count dashboards. Every activation is a trace root. Within a conversation, group by trace ID, order activations by `agent.turn.sequence` or start time, and open the selected trace ID. Child activations link to their caller with `eve.link.type=agent.dispatch`; remote workflows keep their own execution lineage, so use conversation IDs and caller links for cross-deployment correlation.--For example, the built-in `agent` tool dispatches research into a new workflow and trace, and the next user turn starts another trace:--```text-Trace A-invoke_agent support session=S, turn=turn_0, conversation=S- agent.step- chat <model>- agent.action role=caller, action.call_id=C- execute_tool agent--Trace B-invoke_agent research session=R, turn=turn_0, conversation=S- link: agent.dispatch -> Trace A's caller action C- agent.step- chat <model>--Trace C-invoke_agent support session=S, turn=turn_1, conversation=S- agent.step- chat <model>-```--The conversation filter finds all three retained traces. eve initializes the conversation ID once and carries it through the existing parent context for local dispatch and `eve.conversation.id` baggage for remote dispatch. Remote session creation does not carry execution lineage or add authorization requirements for tracing. Existing authentication, principal forwarding, and root-session limits remain unchanged.--Remote dispatch records the caller's span ID in eve's W3C `tracestate` entry because HTTP intermediaries may advance `traceparent` to their own request span. The receiver combines that prior parent with the current trace identity for the causal link and retains the transport context for `channel.request`; adopting either as the child's parent would instead keep both in the same trace. Resolved trace-policy decisions and trusted remote content ceilings continue across this boundary independently of correlation, and an unsampled caller cannot be widened into a sampled child. No synthetic session span is needed.--Activation duration is elapsed time for that activation, including waits inside it, not the lifetime of the conversation or CPU time. Idle time between ended activations is not part of their durations. For token totals, choose one level: filter to `chat` model spans and sum `gen_ai.usage.*`, or filter to activations and sum their `agent.usage.input_tokens` and `agent.usage.output_tokens`. Exclude caller summaries from the activation total. Query cache usage on model spans. Do not add model, step, activation, and caller counters together.--### Search traces--Use your destination's span-search surface to filter by the attributes in the table, group activations by session or trace ID, and open one activation's trace waterfall. Use the OTel span name and `gen_ai.operation.name` to identify the operation. Preserve eve's ownership of the OTel provider; do not register a second provider merely to add a destination.--### Sampling and completeness--A session can contain both retained and missing activation traces. Head sampling, backend retention, destination filtering, and exporter failures all affect what a conversation query returns. Preserve the schema and correlation attributes in Collector transforms and destination policies, and configure indexing and retention for the queries above.--Do not interpret a missing activation as a missing execution, or sampled trace counts and token sums as an exact session ledger. A backend may receive descendants before the activation span finishes and exports; a temporarily missing root is not necessarily a broken parent ID. After deployment, verify a two-turn conversation and a delegation in the actual destination, including names, conversation grouping across services, separate trace roots, caller links, and outcome attributes.--## Runtime context--_Runtime context_ is an [AI SDK concept](https://ai-sdk.dev/docs/reference/ai-sdk-core/stream-text): a user-defined object that flows through a generation lifecycle. eve exposes it through `events["step.started"]`, a callback that runs once eve has assembled the model input for an attempt and returns `{ runtimeContext }`. Because eve registers the AI SDK's OpenTelemetry integration with runtime context enabled, those returned values ride onto the model-call span and its children. The field is named `runtimeContext`, not `metadata`, because AI SDK v7 carries per-call attributes on runtime context rather than a dedicated metadata field.--Use it when the values depend on the current session, turn, step, channel, or model input:--```ts-import { defineInstrumentation, isChannel } from "eve/instrumentation";-import supportChannel from "./channels/support";--export default defineInstrumentation({- events: {- "step.started"(input) {- if (!isChannel(input.channel, supportChannel)) {- return undefined;- }-- return {- runtimeContext: {- "support.channel_id": input.channel.metadata.channelId ?? "",- "support.user_id": input.channel.metadata.triggeringUserId ?? "",- },- };- },- },-});-```--The callback receives:--- `session`: the session id, current and initiator auth, and parent session lineage when this is a child run-- `turn`: the stream turn id and sequence, for example `turn_0`-- `step`: the zero-based step index inside the turn-- `channel`: the channel's `kind` and the metadata projected by the active channel-- `modelInput`: the final instructions and messages passed to the model call--A channel exposes its identity through `kind`. For authored channels it is `channel:<name>`, where `<name>` is the channel's filename under `agent/channels/`, so `agent/channels/support.ts` is `channel:support`. Framework channels use `http`, `schedule`, or `subagent`, and an unrecognized or absent kind normalizes to `unknown`. The kind is also emitted as the `eve.channel.kind` span attribute. To access an authored channel's metadata with its precise type, import the channel definition and narrow with `isChannel(input.channel, supportChannel)`.--Channel metadata is channel-owned. Built-in channels expose only the fields they choose to make observable; Slack, for example, projects `channelId`, `teamId`, `threadTs`, and `triggeringUserId` from its durable channel state. User-authored channels expose their own projection by returning `metadata(state)` from `defineChannel`. Runtime instrumentation never falls back to raw channel state.--## Authored trace hierarchy--When authored telemetry is enabled, each turn currently produces a trace like:--```text-ai.eve.turn {eve.session.id}- +-- invoke_agent <model> gen_ai.operation.name=invoke_agent- +-- step 1 gen_ai.operation.name=agent_step- | +-- chat <model> gen_ai.operation.name=chat- | +-- execute_tool search gen_ai.operation.name=execute_tool- +-- step 2- | +-- chat <model>- | +-- execute_tool read- +-- step 3 (final text)- +-- chat <model>-```--eve creates the `ai.eve.turn` parent span per turn and passes enriched telemetry to the AI SDK so model calls and tool executions are traced automatically. The AI SDK's OpenTelemetry integration names these spans after the [OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/), so backends that understand `gen_ai.operation.name` can classify them without extra configuration. The `invoke_agent` span is named after the model; the agent name is on its `gen_ai.agent.name` attribute.--This hierarchy applies when eve passes telemetry to the AI SDK. When the `otel()` provider layout is declared and eve owns the agent spans, eve names its invocation span `invoke_agent <agent>` and its model-attempt spans `agent.step`. Session, turn, step, and channel context is injected as the framework half of the runtime context (`eve.version`, `eve.session.id`, `eve.environment`, `eve.turn.id`, `eve.turn.sequence`, `eve.step.index`, `eve.channel.kind`) and rides onto the spans alongside any values your `events["step.started"]` callback returns under `runtimeContext`.--Set `traceChannelRequests: true` on `defineInstrumentation` to also wrap each inbound channel HTTP request in a single OpenTelemetry `SERVER` span named for the registered route. In the authored hierarchy above, this span parents the turn tree and any `hook.resume` or outgoing HTTP spans. In the provider layout, `invoke_agent` remains a separate trace root and links to the request span.--```text-POST /eve/v1/session/:sessionId- └── hook.resume- ├── GET hooks/by-token- └── POST hook_received-```--The span stays low-cardinality (route template in `http.route`, method in `http.request.method`, never the concrete URL) and records no session ids, tokens, headers, bodies, or query parameters. It adopts an incoming `traceparent` as its parent when present, so eve requests correlate with upstream traces. It defaults to `false`; enable it only when you want these request spans.--## Workflow run tags--Separately from OpenTelemetry, eve tags every workflow run with reserved `$eve.*` attributes. These live on the Vercel Workflow run, queryable in the Workflow dashboard, not on OTel spans, and you do not configure them: they are framework-owned and emitted automatically on every session, turn, and subagent run, whether or not an `instrumentation.ts` file is present. Authored code cannot set or override the `$eve.` namespace.--They let a dashboard reconstruct the tree of runs behind a single agent invocation and surface model and token usage without reading run bodies.--Structural tags describe each run's place in the tree:--- `$eve.type`: `"session"`, `"turn"`, or `"subagent"`-- `$eve.parent`: session id of the immediate parent-- `$eve.root`: session id of the root session in the chain (group a whole tree with `$eve.root=<id>`)-- `$eve.subagent`: compiled graph node id (subagent runs only)-- `$eve.trigger`: the channel kind that started the run-- `$eve.schedule`: the authored schedule that created the session, including sessions started through a target channel-- `$eve.title`: truncated title derived from the first user message-- `$eve.trace_id`: sampled trace seed available when tagging a session, subagent, or turn run. Use it as a trace link, not a conversation-wide trace identity. Later activations can use other trace IDs; query `gen_ai.conversation.id` for the history. The tag does not confirm that a destination retained the trace.--Per-turn usage tags are written on each step of a turn, accumulating cumulative totals (last write wins):--- `$eve.model`: model id for the turn-- `$eve.input_tokens`, `$eve.output_tokens`, `$eve.cache_read_tokens`: running token counts-- `$eve.tool_count`: number of tools available to the turn--Tag writes are best-effort: a failure is logged once per process and then swallowed, so a broken tag emit never breaks the agent.--These tags power the **Agent Runs** tab in the Vercel dashboard. When you deploy on Vercel, the platform auto-detects `eve` as the framework and surfaces an Agent Runs view under your project's **Observability** tab, where you can browse sessions and drill into each conversation's trace, with no `instrumentation.ts` required. The tab is currently gated per team. See [Deploy to Vercel](./deployment/vercel#inspect-agent-runs) for enablement. Agent Runs is separate from the OpenTelemetry export above. Use OTel when you want spans in Braintrust, PostHog, Sentry, Datadog, or another third-party backend.--## Local traces--Without an `instrumentation.ts`, `eve dev` records spans to disk with one bounded trace per turn, including its model steps and tool calls. Read them two ways:--- [`/traces`](dev-tui#logs-and-traces) in the dev TUI: a live trace viewer that replays captured content as a conversation.-- [`eve traces`](../reference/cli#eve-traces): a span tree in the terminal, `eve traces ls` to list. Works after `eve dev` exits.--Local traces omit model, tool, and memory-record content by default. Set `EVE_TRACES_CONTENT=on` in `.env.local` to capture that content.--Writing `instrumentation.ts` replaces this: your `setup` takes over and nothing is recorded locally. For span attributes, retention, and the `EVE_TRACES*` variables, see [`eve traces`](../reference/cli#eve-traces).--## Debugging--`eve info` is the fastest way to see what eve actually picked up: ordered static instructions with their roles, plus the active tools, skills, subagents, schedules, routes, and discovery diagnostics. Dynamic instruction results exist only at runtime and are not part of this static inspection. eve also writes inspectable artifacts under `.eve/`, kept even when discovery hits errors:--| Artifact | Tells you |-| ------------------------------- | ------------------------------------------- |-| `agent-discovery-manifest.json` | what eve found on disk |-| `diagnostics.json` | authored-shape errors and warnings |-| `compiled-agent-manifest.json` | the serialized surface eve loads at runtime |-| `module-map.mjs` | compiled module entrypoints eve imports |--When `eve build` fails on discovery errors, the CLI prints the full diagnostics report (severity, message, source path) and the path to the diagnostics artifact.--### Common failures--| Symptom | Likely cause and fix |-| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |-| Tool not discovered (the model never sees it) | Run `eve info`. Confirm the file is in the right slot (`agent/tools/<name>.ts`) and default-exports `defineTool(...)`, and check `.eve/diagnostics.json` for shape errors. `schedules/` are root-only. |-| Model won't call a tool it should | Tighten the tool `description` and `inputSchema`; put procedural guidance in a [skill](../skills), not the description. Confirm it's in the active set with `eve info`. |-| Stuck on `session.waiting` | The turn is parked for input. Answer the pending approval or question, or POST a follow-up to `/eve/v1/session/:sessionId`. |-| 401 on production routes | Expected: auth fails closed. Replace `placeholderAuth()` with your route policy. Use `vercelOidc()` only for Vercel-issued tokens; otherwise configure `httpBasic()`, JWT/OIDC helpers, or a custom `AuthFn`. See [Authentication](./auth-and-route-protection). |-| Build fails with discovery errors | Read the printed diagnostics and `.eve/diagnostics.json`; confirm the root-vs-subagent boundary is valid and secrets come from env vars. |--## What to read next--- [`agent.ts`](../agent-config)-- [Hooks](./hooks): observe the runtime event stream-- [Local Development](./dev-tui): drive the agent locally-- [Evals](../evals/overview): repeatable scored checksdiff --git a/docs/guides/instrumentation/meta.json b/docs/guides/instrumentation/meta.jsonnew file mode 100644index 000000000..d8922e16e--- /dev/null+++ b/docs/guides/instrumentation/meta.json@@ -0,0 +1,4 @@+{+ "title": "Observability",+ "pages": ["overview", "providers", "otel"]+}diff --git a/docs/guides/instrumentation/otel.mdx b/docs/guides/instrumentation/otel.mdxnew file mode 100644index 000000000..1a90b394a--- /dev/null+++ b/docs/guides/instrumentation/otel.mdx@@ -0,0 +1,149 @@+---+title: "OpenTelemetry"+description: "Configure OpenTelemetry destinations, content capture, and managed exports with Instrumentation Providers."+url: "/observability/otel"+---++<Callout type="warning" title="Experimental">+ This API is experimental and may change without a deprecation period.+</Callout>++Use the built-in APIs from `eve/instrumentation/otel` to configure eve's+OpenTelemetry pipeline. This page requires the experimental+[Instrumentation Providers](./instrumentation-providers) layout.++## Configure OpenTelemetry++OpenTelemetry configuration has two parts:++- `otel()` declares process-wide settings such as the resource, sampler,+ propagators, and trace capture policy. Declare it at most once.+- `otelIntegration()` adds one destination. Declare one file per exporter or+ processor chain.++Add `agent/instrumentation/otel.ts` when you need process-wide settings or+want to control the content eve writes to OpenTelemetry spans:++```ts title="agent/instrumentation/otel.ts"+import { otel } from "eve/instrumentation/otel";++export default otel({+ resource: { "deployment.environment": process.env.VERCEL_ENV ?? "development" },+ tracePolicy: ({ audience, environment }) => ({+ emit: true,+ recordInputs: audience === "public" || environment === "development",+ recordOutputs: audience === "public" || environment === "development",+ }),+});+```++The OpenTelemetry `tracePolicy` is a capture ceiling shared by every OTel+destination. A destination cannot restore content excluded by this policy. It+does not affect lifecycle-event providers created with `defineInstrumentation()`.++## Add an OpenTelemetry destination++Most agents only need a destination:++```ts title="agent/instrumentation/braintrust.ts"+import { BraintrustExporter } from "@braintrust/otel";+import { otelIntegration } from "eve/instrumentation/otel";++export default otelIntegration({+ traceExporter: new BraintrustExporter({ filterAISpans: true }),+});+```++Pass `spanProcessors` to `otelIntegration()` when a custom destination needs+filtering or transformation. `exportPolicy` is for the built-in+`localTraces()` and `agentRuns()` destinations below.++## Trace topology++OpenTelemetry destinations receive the following trace topology for an ordinary+agent turn:++```text+invoke_agent <agent>+ └── agent.step+ ├── chat <model>+ └── agent.action+ └── execute_tool <tool>+```++Each turn starts a new trace. The first subagent trace links to its caller with+`eve.link.type=agent.dispatch`. Use `gen_ai.conversation.id` to find the traces+for one conversation in your destination's span-search surface.++## Manage built-in destinations++The provider layout has two environment-specific defaults:++- `local` records local traces during `eve dev`.+- `agent-runs` exports to Vercel Agent Runs in preview and production+ deployments.++Omitting these files preserves the defaults. Reconfigure a destination by+exporting `localTraces()` or `agentRuns()` from the matching file. Disable one+explicitly:++```ts title="agent/instrumentation/local.ts"+import { disableInstrumentation } from "eve/instrumentation";++export default disableInstrumentation();+```++## Filter a managed destination++`localTraces()` and `agentRuns()` accept an `exportPolicy`. It filters spans+and attributes before that destination's processors receive them:++```ts title="agent/instrumentation/agent-runs.ts"+import { agentRuns } from "eve/instrumentation/otel";++export default agentRuns({+ exportPolicy: {+ span: ({ name }) => name !== "internal.cache.refresh",+ attribute: ({ key }) => (key === "customer.id" ? { action: "drop" } : { action: "keep" }),+ },+});+```++Return `false` from `span` to omit that span from this destination. Return+`{ action: "keep" }`, `{ action: "drop" }`, or+`{ action: "replace", value }` from `attribute` to retain, remove, or change+one attribute.++### Further narrow a built-in destination++Use the content redactors to further narrow what a built-in destination+receives after the process-wide `otel({ tracePolicy })` has admitted a trace.+For example, keep the spans in Agent Runs while redacting their inputs and+outputs unless the session is public:++```ts title="agent/instrumentation/agent-runs.ts"+import {+ agentRuns,+ composeSpanExportPolicies,+ redactSpanInputs,+ redactSpanOutputs,+} from "eve/instrumentation/otel";++export default agentRuns({+ exportPolicy: composeSpanExportPolicies(+ redactSpanInputs(({ audience }) => audience !== "public"),+ redactSpanOutputs(({ audience }) => audience !== "public"),+ ),+});+```++`redactSpanInputs()` removes eve's known prompt, instruction, document, and+tool-argument attributes. `redactSpanOutputs()` removes response, reasoning,+tool-result, exception, and status attributes. They narrow only this+destination; they do not mutate spans sent to another destination.++## What to read next++- [Instrumentation Providers](./instrumentation-providers): add lifecycle-event providers.+- [Instrumentation](../observability/instrumentation): configure the current single-file API.+- [Local development](../guides/dev-tui): inspect local traces in the TUI.diff --git a/docs/guides/instrumentation/overview.md b/docs/guides/instrumentation/overview.mdnew file mode 100644index 000000000..dafc18821--- /dev/null+++ b/docs/guides/instrumentation/overview.md@@ -0,0 +1,199 @@+---+title: "Instrumentation"+description: "Trace an agent with OpenTelemetry in instrumentation.ts, read the workflow run tags eve emits, and debug discovery with eve info and the common-failures table."+url: "/observability/instrumentation"+---++Use `agent/instrumentation.ts` to configure OpenTelemetry for an eve agent.+eve discovers this file and runs it at server startup before agent code.++## Configure OpenTelemetry++Export `defineInstrumentation(...)` as the default export from+`agent/instrumentation.ts`. eve discovers the file and runs its `setup`+callback at server startup, before agent code runs.++```ts title="agent/instrumentation.ts"+import { BraintrustExporter } from "@braintrust/otel";+import { defineInstrumentation } from "eve/instrumentation";+import { registerOTel } from "@vercel/otel";++export default defineInstrumentation({+ setup: ({ agentName }) =>+ registerOTel({+ serviceName: agentName,+ traceExporter: new BraintrustExporter({+ parent: `project_name:${agentName}`,+ filterAISpans: true,+ }),+ }),+});+```++Use `setup` to register your OTel provider. eve supplies the resolved agent+name, so you do not need to hard-code a service name. Any OTel-compatible+backend works; install and configure the exporter it requires.++eve records metadata without model, tool, or memory-record content by default.+After reviewing the exporter's destination and retention path, opt in to the+content you need:++- `recordInputs` records message history and recalled memory records. It+ defaults to `false`.+- `recordOutputs` records model outputs. It defaults to `false`.+- `functionId` overrides the function name on spans. It defaults to the agent+ name.++The session's channel audience is a second gate on content capture. In+Vercel's preview and production deployments, eve records inputs and outputs+only for `public` sessions. During local development, local tracing retains all+content. See [Local traces](#local-traces) to learn more.++For the default eve HTTP channel, the audience is:++| Session creator | Audience |+| ---------------------------------------------------- | --------- |+| Anonymous caller | `unknown` |+| Authenticated `user`, `service`, or `runtime` caller | `private` |+| Any other principal type | `unknown` |++Set `audience: "public"` only for intentionally public traffic. Other+built-in channels classify their own conversations. See+[Audience](../channels/eve#audience) for the `eve` channel or+[Conversation audience](../channels/custom#conversation-audience) for a+custom channel.++## Add runtime context++Use `events["step.started"]` to add values to the AI SDK runtime context for a+model attempt. The AI SDK carries the returned values onto the model-call span+and its children.++```ts title="agent/instrumentation.ts"+import { defineInstrumentation, isChannel } from "eve/instrumentation";+import supportChannel from "./channels/support";++export default defineInstrumentation({+ events: {+ "step.started"(input) {+ if (!isChannel(input.channel, supportChannel)) return undefined;++ return {+ runtimeContext: {+ "support.channel_id": input.channel.metadata.channelId ?? "",+ "support.user_id": input.channel.metadata.triggeringUserId ?? "",+ },+ };+ },+ },+});+```++The callback receives the session, turn, step, channel, and final model input.+For an authored channel, import its definition and use `isChannel` before+accessing its typed metadata. Runtime instrumentation receives only the+metadata a channel explicitly exposes; it never falls back to raw channel+state.++## Trace topology++When `agent/instrumentation.ts` registers an OTel provider, each turn produces+this hierarchy:++```text+ai.eve.turn {eve.session.id}+ +-- invoke_agent <model> gen_ai.operation.name=invoke_agent+ +-- step 1 gen_ai.operation.name=agent_step+ | +-- chat <model> gen_ai.operation.name=chat+ | +-- execute_tool search gen_ai.operation.name=execute_tool+ +-- step 2+ | +-- chat <model>+ | +-- execute_tool read+ +-- step 3 (final text)+ +-- chat <model>+```++eve creates the `ai.eve.turn` parent span and passes telemetry to the+[AI SDK](https://ai-sdk.dev/docs/ai-sdk-core/telemetry), which traces model+calls and tool executions. The AI SDK names those spans according to the+OpenTelemetry GenAI semantic conventions.++Set `traceChannelRequests: true` to create one OTel `SERVER` span for each+inbound channel HTTP request. The span parents the turn tree and any+`hook.resume` or outgoing HTTP spans. It defaults to `false`.++```text+POST /eve/v1/session/:sessionId+ └── hook.resume+ ├── GET hooks/by-token+ └── POST hook_received+```++The request span uses a route template in `http.route` and the method in+`http.request.method`; it does not record concrete URLs, session IDs, tokens,+headers, bodies, or query parameters. It adopts an incoming `traceparent` when+present.++## Local traces++Without `agent/instrumentation.ts`, `eve dev` records one bounded trace per+turn under `.eve/traces/`. View traces with+[`/traces`](../guides/dev-tui#logs-and-traces) in the dev TUI or+[`eve traces`](../reference/cli#eve-traces) after `eve dev` exits.++Local traces retain model, tool, and memory-record content by default. Set+`EVE_TRACES_CONTENT=off` in `.env.local` to omit it. Writing+`agent/instrumentation.ts` replaces local tracing; its `recordInputs` and+`recordOutputs` settings control content in the OTel provider you register.++## Workflow run tags++Separately from OpenTelemetry, eve tags every Workflow run with reserved+`$eve.*` attributes. These framework-owned attributes are queryable in the+Workflow dashboard, not on OTel spans. eve emits them for every session, turn,+and subagent run, whether or not `agent/instrumentation.ts` exists.++Structural tags describe a run's place in its tree:++- `$eve.type`: `"session"`, `"turn"`, or `"subagent"`.+- `$eve.parent`: the immediate parent session ID.+- `$eve.root`: the root session ID for the tree.+- `$eve.subagent`: the compiled graph node ID for a subagent run.+- `$eve.trigger`: the channel kind that started the run.+- `$eve.schedule`: the schedule that created the session.+- `$eve.title`: a truncated title from the first user message.+- `$eve.trace_id`: a sampled trace seed. Use it as a trace link, not a+ conversation-wide identity.++Each turn also accumulates `$eve.model`, `$eve.input_tokens`,+`$eve.output_tokens`, `$eve.cache_read_tokens`, and `$eve.tool_count`. These+tags power the **Agent Runs** tab in Vercel's **Observability** view. See+[Deploy to Vercel](../guides/deployment/vercel#inspect-agent-runs) for+enablement.++## Debug discovery++Run `eve info` to see the instrumentation eve discovered and any diagnostics.+eve also writes these inspectable artifacts under `.eve/`:++| Artifact | Tells you |+| ------------------------------- | ----------------------------------- |+| `agent-discovery-manifest.json` | What eve found on disk. |+| `diagnostics.json` | Authored-shape errors and warnings. |+| `compiled-agent-manifest.json` | The surface eve loads at runtime. |+| `module-map.mjs` | Compiled module entry points. |++### Common failures++| Symptom | Next action |+| ------------------------------------- | ------------------------------------------------------------------------------------------------- |+| Instrumentation is not discovered. | Run `eve info`, confirm the file is in the expected slot, and read `.eve/diagnostics.json`. |+| A tool is not discovered. | Confirm it is under `agent/tools/`, default-exports `defineTool(...)`, and appears in `eve info`. |+| `eve build` reports discovery errors. | Read the printed diagnostics and `.eve/diagnostics.json`. |++## Instrumentation Providers++[Instrumentation Providers](./instrumentation-providers) is an experimental+multi-file alternative. It cannot be used with `agent/instrumentation.ts`.+Use it when you need independent lifecycle-event providers, OpenTelemetry+destinations, directional content capture, or per-destination redaction.diff --git a/docs/guides/instrumentation/providers.mdx b/docs/guides/instrumentation/providers.mdxnew file mode 100644index 000000000..676d3f027--- /dev/null+++ b/docs/guides/instrumentation/providers.mdx@@ -0,0 +1,169 @@+---+title: "Instrumentation Providers"+description: "Configure experimental instrumentation providers, handle lifecycle events, and control the content each provider receives."+url: "/observability/instrumentation-providers"+---++<Callout type="warning" title="Experimental">+ This API is experimental and may change without a deprecation period.+</Callout>++Instrumentation providers split observability into files under `agent/instrumentation/`. Each provider handles eve lifecycle events without owning the rest of the telemetry pipeline. Configure OpenTelemetry destinations with the [built-in OpenTelemetry APIs](./otel).++Enable it explicitly in `agent.ts`:++```ts title="agent/agent.ts"+import { defineAgent } from "eve";++export default defineAgent({+ model: "anthropic/claude-sonnet-5",+ experimental: {+ instrumentationProviders: true,+ },+});+```++The provider directory replaces `agent/instrumentation.ts`; the two layouts cannot be used together. See [Instrumentation](../observability/instrumentation) for the current single-file API.++## Add a provider++The filename identifies the provider slot. A lifecycle-event provider file+must default-export `defineInstrumentation(...)` or `disableInstrumentation()`.+See [OpenTelemetry](./otel) for the supported OTel declarations.++```text+agent/instrumentation/+ audit.ts lifecycle event provider+```++This provider records action timing and identity without receiving tool arguments or results:++```ts title="agent/instrumentation/audit.ts"+import { defineInstrumentation } from "eve/instrumentation";++export default defineInstrumentation({+ events: {+ "action.started": (event, ctx) => {+ ctx.state.set({ name: event.name, startedAt: Date.now() });+ },+ "action.completed": (event, ctx) => {+ const started = ctx.state.get() as { name: string; startedAt: number } | undefined;+ if (started === undefined) return;++ console.log({+ action: started.name,+ durationMs: Date.now() - started.startedAt,+ outcome: event.outcome,+ });+ },+ },+});+```++`ctx.state` is JSON storage scoped to this provider and operation. It survives durable suspension and is released after the terminal event.++## Control inputs and outputs++Each provider has an independent `tracePolicy`. It decides whether the provider+receives a trace and whether its events include input or output content.++```ts title="agent/instrumentation/audit.ts"+import { defineInstrumentation } from "eve/instrumentation";++export default defineInstrumentation({+ tracePolicy: ({ audience, environment }) => ({+ emit: true,+ recordInputs: audience === "public" || environment === "development",+ recordOutputs: audience === "public" || environment === "development",+ }),+ events: {+ "model.call.started": (event) => {+ console.log("input", event.input);+ },+ "model.call.completed": (event) => {+ console.log("output", event.content);+ },+ },+});+```++The function receives `agentName`, `channel`, `audience`, `mode`,+`environment`, and `principalType`. `audience` is `"public"`, `"private"`, or+`"unknown"`; `environment` is `"development"`, `"preview"`, or+`"production"`.++Without a policy, or when it returns `true`, eve sends the provider metadata+for every trace. It includes input and output content in these cases:++| Environment | Audience | Content |+| --------------------- | ---------------------- | ------------------ |+| Development | Any | Inputs and outputs |+| Preview or production | `public` | Inputs and outputs |+| Preview or production | `private` or `unknown` | Metadata only |++Return `{ emit: false }` to not sample the trace for this provider. Return+`{ emit: true, recordInputs, recordOutputs }` to choose the two content+directions explicitly. Inputs include model prompts, tool arguments, channel+input, and user responses. Outputs include model responses, tool results,+requests for user input, provider metadata, and error details. Omitted content+fields are unavailable to the handler.++The channel assigns the audience once when it creates the session. It controls+content capture, not access. Configure it in the [default eve HTTP+channel](../channels/eve#audience) or a [custom+channel](../channels/custom#conversation-audience).++## Redact fields in a custom provider++Lifecycle events are immutable snapshots. Copy the fields you need into a destination-specific payload and redact that copy before sending it:++```ts title="agent/instrumentation/audit.ts"+import { defineInstrumentation } from "eve/instrumentation";++export default defineInstrumentation({+ tracePolicy: () => ({+ emit: true,+ recordInputs: true,+ recordOutputs: false,+ }),+ events: {+ "action.started": async (event) => {+ await sendAuditRecord({+ id: event.idempotencyKey,+ input: redactApiKey(event.input),+ kind: event.kind,+ name: event.name,+ });+ },+ },+});++function redactApiKey(value: unknown): unknown {+ if (typeof value !== "object" || value === null || Array.isArray(value)) return value;++ const record = value as Record<string, unknown>;+ return "apiKey" in record ? { ...record, apiKey: "[redacted]" } : record;+}++async function sendAuditRecord(record: unknown): Promise<void> {+ // Send the sanitized record to your provider.+ void record;+}+```++Prefer `recordInputs: false` or `recordOutputs: false` when the destination does not need an entire content direction. Use field-level redaction only when the destination needs part of that content.++## Lifecycle events++Providers can handle session, channel delivery, turn, model attempt, model call, action, input request, and tool call events. Start and terminal events share an `idempotencyKey`, which can serve as a destination row ID.++An ordinary tool emits both `action.*` and `tool.call.*` events. Use `action.*` for eve's durable dispatch lifecycle, including tools, skills, subagents, and remote agents. Use `tool.call.*` only when you need the AI SDK's in-process tool execution boundary.++Handlers for different providers run concurrently and are failure-isolated. Do not depend on provider execution order. Use `flush` to drain buffered records and `shutdown` to release resources.++## What to read next++- [Instrumentation](../observability/instrumentation): configure the current single-file API+- [OpenTelemetry](./otel): configure OTel destinations and managed exports.+- [Local development](../guides/dev-tui): inspect local traces in the TUI+- [Hooks](../guides/hooks): react to runtime events outside the instrumentation provider APIdiff --git a/docs/guides/meta.json b/docs/guides/meta.jsonindex 7ec352a9e..e725d4d3a 100644--- a/docs/guides/meta.json+++ b/docs/guides/meta.json@@ -4,7 +4,6 @@"deployment","auth-and-route-protection","instrumentation",- "instrumentation-providers","hooks","session-context","dynamic-capabilities",diff --git a/docs/guides/remote-agents.md b/docs/guides/remote-agents.mdindex 9565e3c72..6b06f12b2 100644--- a/docs/guides/remote-agents.md+++ b/docs/guides/remote-agents.md@@ -139,26 +139,22 @@ Identity forwarding does not make a persistent session private to one caller. CoForwarding is explicit on both sides. The receiver names which forwarders it trusts with `eveChannel({ trustedForwarders })` (see [Auth & route protection](./auth-and-route-protection#accepting-forwarded-identity-from-another-deployment)); a receiver that refuses the forwarder — or has no `trustedForwarders` at all — rejects with a 403 and the dispatch fails.-Remote tracing uses [W3C Trace Context](https://www.w3.org/TR/trace-context/). `traceparent` identifies the immediate HTTP parent, while eve's `eve=<caller-span-id>` vendor entry in `tracestate` preserves the dispatching caller when an intermediary advances `traceparent` to its own request span. The receiver uses the eve entry for the child's `agent.dispatch` link and the transport context for request correlation. Older senders and missing or malformed eve entries fall back to `traceparent`. Trace context is not an authorization grant: it cannot assert eve parent lineage, change `rootSessionId`, or remove the normal root-session token cap.+## Trace propagation-eve also carries the original `gen_ai.conversation.id` in `eve.conversation.id` baggage so you can find related traces across local and remote agents. This observability identifier does not require principal forwarding or shared execution lineage; it grants no session access and does not change trace-content policy. Session execution establishes this ID even without an instrumentation runtime. Only session-create requests with a callback may supply this baggage; top-level requests ignore it and establish their own conversation ID. Callback metadata is caller-supplied correlation, not verified identity.--eve replaces configured `traceparent` and conversation baggage only when it has a framework value to send. It replaces its own `tracestate` entry while preserving other valid vendor entries; without a valid caller context, eve removes only its entry. Incoming baggage has an 8 KiB limit; the audience and conversation readers share whitespace, percent-decoding, and duplicate-key validation. Conversation IDs are limited to 1 KiB and exclude control characters and line separators. If adding an audience assertion or conversation ID would exceed 8 KiB, eve rejects the dispatch before sending a request instead of dropping the member. Reduce `remote.headers.baggage` to leave room for these framework values.--In the [provider trace contract](./instrumentation#agent-trace-contract), each-child activation starts a separate trace. The first activation uses the-caller preserved in W3C `tracestate`, falling back to incoming `traceparent`,-as an `agent.dispatch` span link rather than adopting the caller's trace ID.--> ⚠️ **Upgrade both deployments before resuming persistent remote sessions.** A sender with continuation forwarding includes `forwardedPrincipal` on each authenticated follow-up. A receiver that supports forwarding only on session creation rejects that continuation with HTTP 400. eve does not retry without the field because that would run the follow-up as the transport service principal and silently change caller authority. The parent retains the child handle after this failure, so you can retry the same session after upgrading the receiver.--A receiver on an eve version that predates all principal forwarding may instead drop the unknown field and run the session as your app's service identity; per-user connections there fail with `principal_required`. On remote requests where the dispatching turn has no auth, the field is omitted and the call proceeds on transport trust alone.+Each remote turn starts a new trace. eve links the child trace to the+dispatching turn and carries `gen_ai.conversation.id` so you can find the+traces for one conversation. Trace context is observability metadata, not an+authorization grant. See [OpenTelemetry](../observability/otel#trace-topology)+for the trace topology.## Preserving trace content-With `forwardPrincipal: true`, a sampled trace carries its original audience and the maximum content the next hop may record. For example, `eve.audience=private;ceiling=i0o1` allows outputs but not inputs. eve sends this as [W3C Baggage](https://www.w3.org/TR/baggage/).+With `forwardPrincipal: true`, a sampled remote dispatch forwards its original+audience and the maximum input and output content the next hop may record. eve+sends this policy as [W3C Baggage](https://www.w3.org/TR/baggage/).-The receiver uses it only after `trustedForwarders` accepts the authenticated calling deployment:+The receiving deployment uses the policy only after it trusts the calling+deployment:```ts title="agent/channels/eve.ts"import { eveChannel } from "eve/channels/eve";@@ -171,13 +167,15 @@ export default eveChannel({});```-The request must also include a callback and a valid sampled `traceparent`. Those fields identify a remote call, but they do not establish trust. `trustedForwarders` is the authorization boundary.--The receiver combines the incoming ceiling with its own trace policy. Each hop may narrow the result, but it cannot restore inputs or outputs removed earlier. The original audience stays the same across remote and local subagent hops. Public origins may include content by default. Private and unknown origins stay metadata-only in preview and production unless both deployments explicitly allow them; a receiver running in development still applies the development content default.--The live delivery audience still matters. An unknown callback delivery, or one matching the origin, uses the session decision. A different explicit audience applies its own hard ceiling, so a private delivery stays redacted even when the trace began in public.+The request must include a callback and a valid sampled `traceparent`.+`trustedForwarders` is the authorization boundary. When the assertion is+accepted, the receiver uses the forwarded audience instead of reclassifying+the child session with its local channel.-Missing, malformed, duplicate, unsampled, untrusted, and mixed-version assertions fall back to metadata-only tracing. Dropped traces use only the unsampled trace flag. The decision is fixed when the remote session starts and reused by continuations. Agent Runs shows Workflow content only when both inputs and outputs are allowed.+The receiver combines the forwarded ceiling with its own trace policy. Each+hop may narrow content capture, but cannot restore inputs or outputs removed by+an earlier hop. Missing, malformed, unsampled, or untrusted assertions do not+widen capture and use metadata-only tracing.## How remote dispatch and callbacks workdiff --git a/docs/reference/cli.md b/docs/reference/cli.mdindex bd40fac96..a4d400d48 100644--- a/docs/reference/cli.md+++ b/docs/reference/cli.md@@ -317,7 +317,7 @@ A durable conversation produces one bounded trace per turn. Worker replacementsEvery span carries a real duration. A turn's root `invoke_agent` span is written when the turn settles, so a running turn shows only its steps.-Model, `execute_tool`, and memory spans omit their content by default. Set `EVE_TRACES_CONTENT=on` to capture system prompts, prompt messages, and response text for models; call arguments and results for tools; and recalled memory records. Each captured value is capped at 32 KB.+Model, `execute_tool`, and memory spans retain their content by default. Set `EVE_TRACES_CONTENT=off` to omit system prompts, prompt messages, and response text for models; call arguments and results for tools; and recalled memory records. Each captured value is capped at 32 KB.Step spans carry token counts under `agent.usage.*`, and cost when Vercel AI Gateway served the call. Model spans also expose `gen_ai.usage.*` token counters. The CLI sums step-level counters only, so model and delegated-call totals are not counted twice.@@ -325,13 +325,13 @@ Step spans carry token counts under `agent.usage.*`, and cost when Vercel AI Gateve sweeps the store when an activation's writes finish, when a session finishes, and when the dev server starts. An open conversation does not pin every completed turn's trace. Sweeps evict oldest-first past the bounds below, except that active traces, the newest traces, and anything written in the last five minutes are kept. A sweep can therefore exceed the size budget. Set the bounds in `.env.local`, which `eve dev` loads automatically; each accepts `off` to disable it individually.-| Variable | Default | Effect |-| ---------------------------- | -------------------- | --------------------------------------------------------------------------------------------------- |-| `EVE_TRACES` | on | `off` stops writing traces and stops sweeping |-| `EVE_TRACES_CONTENT` | off | `on` captures model prompt/response, tool input/output, and memory-record attributes on local spans |-| `EVE_TRACES_MAX_AGE_MS` | `604800000` (7d) | Age after which a trace may be evicted |-| `EVE_TRACES_MAX_TOTAL_BYTES` | `536870912` (512 MB) | Size budget for the whole store |-| `EVE_TRACES_RETAIN_COUNT` | `20` | Newest traces kept regardless of age or size |+| Variable | Default | Effect |+| ---------------------------- | -------------------- | ------------------------------------------------------------------------------------------------- |+| `EVE_TRACES` | on | `off` stops writing traces and stops sweeping |+| `EVE_TRACES_CONTENT` | on | `off` omits model prompt/response, tool input/output, and memory-record attributes on local spans |+| `EVE_TRACES_MAX_AGE_MS` | `604800000` (7d) | Age after which a trace may be evicted |+| `EVE_TRACES_MAX_TOTAL_BYTES` | `536870912` (512 MB) | Size budget for the whole store |+| `EVE_TRACES_RETAIN_COUNT` | `20` | Newest traces kept regardless of age or size |## `eve link`@@ -399,10 +399,10 @@ Lists the user-authored channels in the current project.4. `eve build` before shipping.5. `eve start` to smoke-test the built output locally.-Related: [Project layout](../getting-started#project-layout) · [instrumentation.ts](../guides/instrumentation).+Related: [Project layout](../getting-started#project-layout) · [Instrumentation](../observability/instrumentation).## What to read next- [Project layout](../getting-started#project-layout): what `eve info` discovers-- [instrumentation.ts](../guides/instrumentation): tracing and the error catalog+- [Instrumentation](../observability/instrumentation): tracing and the error catalog- [Deployment](../guides/deployment/overview): `eve build` and `eve start` in productiondiff --git a/docs/reference/typescript-api.md b/docs/reference/typescript-api.mdindex a0fceb87c..d8feff553 100644--- a/docs/reference/typescript-api.md+++ b/docs/reference/typescript-api.md@@ -48,7 +48,7 @@ export default defineTool({| `defineSchedule` | `eve/schedules` | `agent/schedules/<name>.ts` | [Schedules](../schedules) || `defineState` | `eve/context` | tools, hooks, lifecycle | [Session context](../guides/session-context) || `defineSandbox` | `eve/sandbox` | `agent/sandbox.ts` | [Sandbox](../sandbox) |-| `defineInstrumentation` | `eve/instrumentation` | `agent/instrumentation.ts` | [instrumentation.ts](../guides/instrumentation) |+| `defineInstrumentation` | `eve/instrumentation` | `agent/instrumentation.ts` | [Instrumentation](../observability/instrumentation) || `defineRemoteAgent` | `eve` | `agent/subagents/<id>/agent.ts` | [Remote agents](../guides/remote-agents) || `defineEval` | `eve/evals` | `evals/*.eval.ts` | [Evals](../evals/overview) || `defineEvalConfig` | `eve/evals` | `evals/evals.config.ts` | [Evals](../evals/overview) |