The Open Framework for Building Agents
feat(eve): add runtimeContext to otelIntegration and instrumentations to otel (#2112)
Chad Hietalae306fc342213
diff --git a/packages/eve/src/harness/instrumentation/providers.ts b/packages/eve/src/harness/instrumentation/providers.tsindex 7217e4205..73b119e5c 100644--- a/packages/eve/src/harness/instrumentation/providers.ts+++ b/packages/eve/src/harness/instrumentation/providers.ts@@ -115,10 +115,13 @@ export function finalizeInstrumentationProviders(input: {readonly serviceName: string;}): InstrumentationRuntime {const registered = getInstrumentationProviders();+ const providerDefinitions = registered.map(toProviderDefinition);+ const collected = collectOtelPipeline(registered.map((entry) => entry.provider));return installInstrumentationRuntime({- collected: collectOtelPipeline(registered.map((entry) => entry.provider)),+ collected,frameworkVersion: resolveInstalledPackageInfo().version,- providers: registered.map(toProviderDefinition),+ providers: providerDefinitions,+ runtimeContextResolvers: collected.runtimeContextResolvers,serviceName: input.serviceName,});}diff --git a/packages/eve/src/harness/instrumentation/runtime-context.test.ts b/packages/eve/src/harness/instrumentation/runtime-context.test.tsindex 9e392cf4d..ed48f80db 100644--- a/packages/eve/src/harness/instrumentation/runtime-context.test.ts+++ b/packages/eve/src/harness/instrumentation/runtime-context.test.ts@@ -8,6 +8,7 @@ import {buildTelemetryRuntimeContext,type BuildTelemetryRuntimeContextInput,} from "#harness/instrumentation/runtime-context.js";+import type { RuntimeContextResolver } from "#tracing/otel-declaration.js";import type { HarnessSession } from "#harness/types.js";import type {InstrumentationStepStartedEventInput,@@ -231,4 +232,61 @@ describe("buildTelemetryRuntimeContext", () => {}expect(captured.channel.metadata).toMatchObject({ nested: { value: "original" } });});++ describe("provider runtimeContext resolvers", () => {+ it("emits framework keys when a provider resolver returns undefined", () => {+ const resolver: RuntimeContextResolver = () => undefined;+ const runtimeContext = build({ authored: undefined, providerResolvers: [resolver] });++ expect(runtimeContext).toEqual(FRAMEWORK_KEYS);+ });++ it("merges a single provider resolver beneath framework keys", () => {+ const resolver: RuntimeContextResolver = () => ({ team: "platform" });+ const runtimeContext = build({ authored: undefined, providerResolvers: [resolver] });++ expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, team: "platform" });+ });++ it("merges multiple provider resolvers, later ones overriding earlier", () => {+ const first: RuntimeContextResolver = () => ({ env: "prod", team: "a" });+ const second: RuntimeContextResolver = () => ({ team: "b" });+ const runtimeContext = build({ authored: undefined, providerResolvers: [first, second] });++ expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, env: "prod", team: "b" });+ });++ it("drops reserved eve.* keys from provider resolver results", () => {+ const resolver: RuntimeContextResolver = () =>+ ({ "eve.session.id": "override", team: "platform" }) as never;+ const runtimeContext = build({ authored: undefined, providerResolvers: [resolver] });++ expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, team: "platform" });+ });++ it("continues when a provider resolver throws", () => {+ const failing: RuntimeContextResolver = () => {+ throw new Error("boom");+ };+ const healthy: RuntimeContextResolver = () => ({ team: "platform" });+ const runtimeContext = build({ authored: undefined, providerResolvers: [failing, healthy] });++ expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, team: "platform" });+ });++ it("returns undefined when no authored config and no provider resolvers", () => {+ expect(build({ authored: undefined, providerResolvers: undefined })).toBeUndefined();+ expect(build({ authored: undefined, providerResolvers: [] })).toBeUndefined();+ });++ it("merges provider resolver results alongside the legacy step.started hook", () => {+ const resolver: RuntimeContextResolver = () => ({ source: "provider" });+ const runtimeContext = build({+ authored: { events: { "step.started": () => ({ runtimeContext: { source: "legacy" } }) } },+ providerResolvers: [resolver],+ });++ expect(runtimeContext).toEqual({ ...FRAMEWORK_KEYS, source: "provider" });+ });+ });});diff --git a/packages/eve/src/harness/instrumentation/runtime-context.ts b/packages/eve/src/harness/instrumentation/runtime-context.tsindex 2cee9fe84..64887d651 100644--- a/packages/eve/src/harness/instrumentation/runtime-context.ts+++ b/packages/eve/src/harness/instrumentation/runtime-context.ts@@ -11,6 +11,7 @@ import {} from "#context/keys.js";import type { HarnessEmissionState } from "#harness/emission.js";import type { HarnessSession } from "#harness/types.js";+import type { RuntimeContextResolver } from "#tracing/otel-declaration.js";import {normalizeInstrumentationChannelKind,resolveInstrumentationProjection,@@ -35,6 +36,7 @@ export interface BuildTelemetryRuntimeContextInput {readonly instructions: string | SystemModelMessage | undefined;readonly messages: readonly ModelMessage[];};+ readonly providerResolvers?: readonly RuntimeContextResolver[];readonly session: HarnessSession;}@@ -48,16 +50,21 @@ export interface BuildTelemetryRuntimeContextInput {export function buildTelemetryRuntimeContext(input: BuildTelemetryRuntimeContextInput,): Record<string, unknown> | undefined {- if (input.authored === undefined) {+ const hasAuthored = input.authored !== undefined;+ const hasProviderResolvers =+ input.providerResolvers !== undefined && input.providerResolvers.length > 0;+ if (!hasAuthored && !hasProviderResolvers) {return undefined;}const authoredRuntimeContext = resolveStepStartedRuntimeContext(input);+ const providerRuntimeContext = resolveProviderRuntimeContext(input);const context = contextStorage.getStore();const projection = context?.get(ChannelInstrumentationKey);return {...authoredRuntimeContext,+ ...providerRuntimeContext,"eve.channel.kind": normalizeInstrumentationChannelKind(projection?.kind),"eve.environment": input.environment,"eve.session.id": input.session.sessionId,@@ -160,6 +167,46 @@ function resolveStepStartedRuntimeContext(return filterAuthoredRuntimeContext(runtimeContext, source);}+/**+ * Collects `runtimeContext` contributions from every provider resolver,+ * invoking each with a snapshot of the same input the legacy `step.started`+ * hook receives. Failures are warning-only so one provider cannot break the+ * turn. Later providers override earlier ones on key collision.+ */+function resolveProviderRuntimeContext(+ input: BuildTelemetryRuntimeContextInput,+): InstrumentationRuntimeContext | undefined {+ const resolvers = input.providerResolvers;+ if (resolvers === undefined || resolvers.length === 0) {+ return undefined;+ }++ const merged: Record<string, JsonValue> = {};+ let contributed = false;++ for (const resolver of resolvers) {+ const source = "provider.runtimeContext";+ const invoke = () => {+ const stepStartedInput = buildInstrumentationStepStartedInput(input);+ return resolver(stepStartedInput);+ };+ const result = resolveInstrumentationProjection({+ invoke,+ log,+ source,+ });+ if (result === undefined) continue;++ const filtered = filterAuthoredRuntimeContext(result as JsonObject, source);+ if (filtered === undefined) continue;++ Object.assign(merged, filtered);+ contributed = true;+ }++ return contributed ? merged : undefined;+}+function projectSessionAuth(context: AlsContext | undefined): {readonly current: SessionAuthContext | null;readonly initiator: SessionAuthContext | null;diff --git a/packages/eve/src/harness/instrumentation/runtime.ts b/packages/eve/src/harness/instrumentation/runtime.tsindex 55de6bbb1..188ca6aa8 100644--- a/packages/eve/src/harness/instrumentation/runtime.ts+++ b/packages/eve/src/harness/instrumentation/runtime.ts@@ -5,7 +5,7 @@ import type {InstrumentationTraceContext,InstrumentationTurnStartedEvent,} from "#harness/instrumentation/lifecycle.js";-import type { OtelHarnessSettings } from "#tracing/otel-declaration.js";+import type { OtelHarnessSettings, RuntimeContextResolver } from "#tracing/otel-declaration.js";const INSTRUMENTATION_RUNTIME_KEY = Symbol.for("eve.instrumentation-runtime");@@ -20,6 +20,8 @@ export interface InstrumentationRuntime {event: InstrumentationTurnStartedEvent,) => Promise<InstrumentationTraceContext>;otelSettings: OtelHarnessSettings | undefined;+ /** Provider `runtimeContext` resolvers, collected at install time. */+ readonly runtimeContextResolvers?: readonly RuntimeContextResolver[];readonly runInContext: InstrumentationContextRunner;readonly shutdown: () => Promise<void>;}diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex 314a23504..a0b4ebb21 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -558,11 +558,13 @@ function buildHarnessToolsWithDynamicSubagents(export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {const baseEmit = config.handleEvent;- const otelSettings = getInstrumentationRuntime()?.otelSettings;- // The custom-context enrichment below still reads the authored config object- // directly. Its replacement on the provider surface is unresolved, so a- // provider directory contributes no custom context yet.+ const instrumentationRuntime = getInstrumentationRuntime();+ const otelSettings = instrumentationRuntime?.otelSettings;+ // The legacy single-file layout reads its runtime-context resolver from the+ // authored config object; the provider directory collects resolvers at install+ // time onto the runtime. Both paths feed buildTelemetryRuntimeContext.const authoredConfig = getInstrumentationConfig();+ const providerRuntimeContextResolvers = instrumentationRuntime?.runtimeContextResolvers;if (otelSettings !== undefined) {ensureOtelIntegration();}@@ -1289,6 +1291,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {instructions,messages: modelMessages,},+ providerResolvers: providerRuntimeContextResolvers,session,}),};diff --git a/packages/eve/src/public/instrumentation/index.ts b/packages/eve/src/public/instrumentation/index.tsindex 44986d4b3..77d604538 100644--- a/packages/eve/src/public/instrumentation/index.ts+++ b/packages/eve/src/public/instrumentation/index.ts@@ -92,9 +92,9 @@ export interface InstrumentationModelInput {}/**- * Input passed to `events["step.started"]`. eve runs the callback after- * building the final model input for this attempt and before constructing- * the AI SDK model call.+ * Input passed to `events["step.started"]` and to a provider's+ * `runtimeContext` resolver. eve builds it after assembling the final model+ * input for this attempt and before constructing the AI SDK model call.*/export interface InstrumentationStepStartedEventInput {readonly channel: InstrumentationChannel;@@ -104,6 +104,13 @@ export interface InstrumentationStepStartedEventInput {readonly turn: InstrumentationTurn;}+/**+ * Input passed to a provider's `runtimeContext` resolver. Same shape as+ * {@link InstrumentationStepStartedEventInput}: channel, session, model input,+ * step, and turn coordinates.+ */+export type InstrumentationRuntimeContextInput = InstrumentationStepStartedEventInput;+/*** Result of a `step.started` callback. eve merges `runtimeContext` into the* AI SDK telemetry span; child spans inherit the values. Keys beginning withdiff --git a/packages/eve/src/tracing/install-instrumentation-runtime.ts b/packages/eve/src/tracing/install-instrumentation-runtime.tsindex 6cdb015ac..c3882e7c5 100644--- a/packages/eve/src/tracing/install-instrumentation-runtime.ts+++ b/packages/eve/src/tracing/install-instrumentation-runtime.ts@@ -13,7 +13,7 @@ import { createLogger, formatError } from "#internal/logging.js";import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";import { hasSessionRelease, type LocalTracesProcessor } from "#tracing/local-traces.js";-import type { CollectedOtel } from "#tracing/otel-declaration.js";+import type { CollectedOtel, RuntimeContextResolver } from "#tracing/otel-declaration.js";import { registerOtelPipeline, type RegisteredOtelPipeline } from "#tracing/otel-registration.js";const log = createLogger("tracing.install-instrumentation-runtime");@@ -32,6 +32,7 @@ export function installInstrumentationRuntime(input: {readonly collected: CollectedOtel;readonly frameworkVersion: string;readonly providers: readonly InstrumentationProviderDefinition[];+ readonly runtimeContextResolvers?: readonly RuntimeContextResolver[];readonly serviceName: string;}): InstrumentationRuntime {const serialBefore: InstrumentationProviderDefinition[] = [];@@ -82,6 +83,7 @@ export function installInstrumentationRuntime(input: {otelSettings: input.collected.declared ? input.collected.settings : undefined,prepareSessionTrace,prepareTurnTrace,+ runtimeContextResolvers: input.runtimeContextResolvers,runInContext,shutdown: () => {shutdown ??= settleAll([diff --git a/packages/eve/src/tracing/otel-declaration.test.ts b/packages/eve/src/tracing/otel-declaration.test.tsindex b0267e878..89789075d 100644--- a/packages/eve/src/tracing/otel-declaration.test.ts+++ b/packages/eve/src/tracing/otel-declaration.test.ts@@ -186,4 +186,17 @@ describe("collectOtelPipeline", () => {collectOtelPipeline([otel({ sampler: "always_on" }), otel({ sampler: "always_off" })]),).toThrow(/declares `otel\(\)` more than once/u);});++ it("passes declared instrumentations onto the pipeline", () => {+ const instrumentation = { name: "test-instrumentation" };+ const collected = collectOtelPipeline([otel({ instrumentations: [instrumentation] })]);++ expect(collected.pipeline.instrumentations).toStrictEqual([instrumentation]);+ });++ it("defaults to undefined instrumentations when none are declared", () => {+ const collected = collectOtelPipeline([otel()]);++ expect(collected.pipeline.instrumentations).toBeUndefined();+ });});diff --git a/packages/eve/src/tracing/otel-declaration.ts b/packages/eve/src/tracing/otel-declaration.tsindex bb1bdb6fe..3d7e8437f 100644--- a/packages/eve/src/tracing/otel-declaration.ts+++ b/packages/eve/src/tracing/otel-declaration.ts@@ -7,6 +7,8 @@ import type {} from "#compiled/@vercel/otel/index.js";import { PROVIDER, type InstrumentationProvider } from "#public/instrumentation/provider.js";+import type { InstrumentationRuntimeContextInput } from "#public/instrumentation/index.js";+import type { JsonObject } from "#shared/json.js";import { batchSpanProcessor } from "#tracing/batch-span-processor.js";import type { ResolvedContentOptions } from "#tracing/content-attributes.js";import { contentFilteringProcessor } from "#tracing/content-span-processor.js";@@ -19,9 +21,11 @@ import { vercelRuntimeSpanProcessor } from "#tracing/vercel-runtime-span-exporte* one tracer provider, so it has one resource, one sampler, and one propagator* set. Destinations are the plural half and live in `otelIntegration()`.*- * `contextManager` and `instrumentations` are deliberately absent: eve's span- * nesting depends on the former, and the latter cannot reach anything eve- * imported before setup ran.+ * `contextManager` is deliberately absent: eve's span nesting depends on it.+ * `instrumentations` is accepted so providers can opt into Node auto-+ * instrumentations (e.g. `@opentelemetry/auto-instrumentations-node`); the+ * packages patch modules eve already imported, so their effects are limited+ * to code loaded after registration.*/export interface OtelOptions {/**@@ -49,6 +53,15 @@ export interface OtelOptions {readonly sampler?: SamplerOrName;/** Composed into one propagator. All inject; the first to extract wins. */readonly propagators?: readonly PropagatorOrName[];+ /**+ * OpenTelemetry `Instrumentation` instances passed through to+ * `registerOTel`. Use them to patch Node.js built-ins (HTTP, DNS, fs, etc.)+ * for automatic spans around outbound work. Disabled by default because eve+ * already imports the model SDK before registration, so patching cannot+ * reach it — but code loaded after registration (tool modules, connection+ * clients) will be instrumented.+ */+ readonly instrumentations?: readonly unknown[];}/**@@ -73,6 +86,17 @@ export interface OtelIntegrationOptions extends ContentOptions {readonly spanProcessors?: readonly SpanProcessor[];/** Wrapped in eve's batching processor and appended after `spanProcessors`. */readonly traceExporter?: SpanExporter;+ /**+ * Contributes runtime context that the AI SDK merges into telemetry spans+ * for each model call. Child spans inherit the values, so a destination can+ * stamp channel or auth identity onto every span in the turn.+ *+ * Synchronous: the harness collects from every destination before the model+ * call, so a return that is not a plain object is dropped (warning-only).+ * Keys beginning with `eve.` are reserved and dropped. Return `undefined`+ * to contribute nothing.+ */+ readonly runtimeContext?: (input: InstrumentationRuntimeContextInput) => JsonObject | undefined;}const OTEL_DECLARATION = Symbol.for("eve.instrumentation.otel");@@ -92,6 +116,7 @@ export interface OtelIntegration extends InstrumentationProvider {readonly [OTEL_INTEGRATION]: true;/** Resolved from `ContentOptions`, so the union does not re-apply defaults. */readonly content: ResolvedContentOptions;+ readonly runtimeContext?: (input: InstrumentationRuntimeContextInput) => JsonObject | undefined;readonly spanProcessors: readonly SpanProcessorOrName[];}@@ -132,6 +157,7 @@ export function otelIntegration(options: OtelIntegrationOptions = {}): OtelInteg[OTEL_INTEGRATION]: true,[PROVIDER]: true,content,+ runtimeContext: options.runtimeContext,spanProcessors:content.recordInputs && content.recordOutputs? spanProcessors@@ -178,6 +204,7 @@ export function isOtelIntegration(value: unknown): value is OtelIntegration {/** The one pipeline a process can register. @internal */export interface OtelPipeline {+ readonly instrumentations?: readonly unknown[];readonly propagators?: readonly PropagatorOrName[];readonly resource?: Readonly<Record<string, unknown>>;readonly sampler?: SamplerOrName;@@ -197,6 +224,11 @@ export interface OtelHarnessSettings {readonly recordOutputs?: boolean;}+/** @internal */+export type RuntimeContextResolver = (+ input: InstrumentationRuntimeContextInput,+) => JsonObject | undefined;+/** @internal */export interface CollectedOtel {/**@@ -205,6 +237,7 @@ export interface CollectedOtel {*/readonly declared: boolean;readonly pipeline: OtelPipeline;+ readonly runtimeContextResolvers: readonly RuntimeContextResolver[];readonly settings: OtelHarnessSettings;}@@ -224,6 +257,7 @@ export interface CollectedOtel {*/export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {const spanProcessors: SpanProcessorOrName[] = [];+ const runtimeContextResolvers: RuntimeContextResolver[] = [];let declaration: OtelDeclaration | undefined;let declared = false;let recordInputs = false;@@ -235,6 +269,9 @@ export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {recordInputs ||= value.content.recordInputs;recordOutputs ||= value.content.recordOutputs;spanProcessors.push(...value.spanProcessors);+ if (value.runtimeContext !== undefined) {+ runtimeContextResolvers.push(value.runtimeContext);+ }continue;}if (!isOtelDeclaration(value)) continue;@@ -251,11 +288,13 @@ export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {return {declared,pipeline: {+ instrumentations: options.instrumentations,propagators: options.propagators,resource: options.resource,sampler: options.sampler,spanProcessors,},+ runtimeContextResolvers,settings: {functionId: options.functionId,recordInputs,diff --git a/packages/eve/src/tracing/otel-registration.ts b/packages/eve/src/tracing/otel-registration.tsindex e9e02950d..7edd51227 100644--- a/packages/eve/src/tracing/otel-registration.ts+++ b/packages/eve/src/tracing/otel-registration.ts@@ -119,9 +119,7 @@ export function registerOtelPipeline(input: {attributes: pipeline.resource,autoDetectResources: false,idGenerator,- // eve imports the model SDK before any of this runs, so an auto- // instrumentation registered here could not patch it anyway.- instrumentations: [],+ instrumentations: pipeline.instrumentations ?? [],propagators: [...(pipeline.propagators ?? ["none"]), markerPropagator],serviceName: input.serviceName,spanProcessors: pipeline.spanProcessors.map((processor) =>