The Open Framework for Building Agents
feat(eve): trace durable channel deliveries (#1935)
Chad Hietala82c13147df82
diff --git a/.changeset/channel-delivery-lifecycle.md b/.changeset/channel-delivery-lifecycle.mdnew file mode 100644index 000000000..1564a96a4--- /dev/null+++ b/.changeset/channel-delivery-lifecycle.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Instrumentation providers can observe durable inbound channel deliveries from processing start through the resulting turn terminal. OpenTelemetry destinations receive consumer spans under the session window, with traced HTTP requests represented as links rather than parents.diff --git a/docs/guides/instrumentation.md b/docs/guides/instrumentation.mdindex 92dc48b4e..63ba41822 100644--- a/docs/guides/instrumentation.md+++ b/docs/guides/instrumentation.md@@ -58,6 +58,28 @@ You are responsible for ensuring any observability or eval provider is approvedThe third configurable surface, [runtime context events](#runtime-context), attaches per-model-call values to these spans.+## Channel delivery traces++Instrumentation providers receive `channel.delivery.started` followed by+`channel.delivery.completed`, `channel.delivery.cancelled`, or+`channel.delivery.failed` for every inbound channel operation. The lifecycle+covers durable processing through the terminal state of the resulting turn, not+messages an adapter sends back to Slack, Telegram, Twilio, or another platform.+Several deliveries can coalesce into one turn while retaining separate lifecycle+pairs, and an adapter can consume a delivery without starting a turn.++Each operation has a framework-owned `deliveryId` distinct from its optional+platform request ID. Metadata-only providers receive identity, channel, session,+and outcome fields. Content providers additionally receive only eve's known+message, context, input-response, and output-schema fields; adapter-specific+payload fields are never projected.++The built-in OpenTelemetry provider maps each pair to an+`agent.channel.delivery` consumer span under the durable session window. When+`traceChannelRequests: true` creates an inbound HTTP server span, the delivery+span links to it with `eve.link.type=channel.request` rather than using the+short-lived request span as its parent.+## Runtime context_Runtime context_ is an [AI SDK concept](https://ai-sdk.dev/docs/reference/ai-sdk-core/stream-text): a user-defined object that flows through a generation lifecycle. eve exposes it through `events["step.started"]`, a callback that runs once eve has assembled the model input for an attempt and returns `{ runtimeContext }`. Because eve registers the AI SDK's OpenTelemetry integration with runtime context enabled, those returned values ride onto the model-call span and its children. The field is named `runtimeContext`, not `metadata`, because AI SDK v7 carries per-call attributes on runtime context rather than a dedicated metadata field.diff --git a/packages/eve/src/channel/channel-address.test.ts b/packages/eve/src/channel/channel-address.test.tsindex ffcd8999a..6ed63f55f 100644--- a/packages/eve/src/channel/channel-address.test.ts+++ b/packages/eve/src/channel/channel-address.test.ts@@ -86,6 +86,36 @@ describe("createChannelAddress", () => {);});+ it("mints a distinct delivery identity for each channel operation", async () => {+ const runtime = createRuntime();+ const address = createChannelAddress({+ adapter: { kind: "slack" },+ channelName: "slack",+ continuationToken: "C1:T1",+ metadata: {+ channelKind: "channel:slack",+ channelName: "slack",+ requestId: "request-1",+ },+ runtime,+ });++ await address.send("first", { auth: null });+ await address.send("second", { auth: null });++ const calls = vi.mocked(runtime.dispatchContinuation).mock.calls;+ const first = calls[0]?.[0].command;+ const second = calls[1]?.[0].command;+ expect(first?.kind === "send" ? first.delivery : undefined).toMatchObject({+ channelKind: "channel:slack",+ channelName: "slack",+ requestId: "request-1",+ });+ expect(first?.kind === "send" ? first.delivery?.deliveryId : undefined).not.toBe(+ second?.kind === "send" ? second.delivery?.deliveryId : undefined,+ );+ });+it("binds every control directly to the namespaced continuation token", async () => {const runtime = createRuntime();const address = createChannelAddress({diff --git a/packages/eve/src/channel/channel-address.ts b/packages/eve/src/channel/channel-address.tsindex bac4abb28..9e30aa384 100644--- a/packages/eve/src/channel/channel-address.ts+++ b/packages/eve/src/channel/channel-address.ts@@ -1,6 +1,10 @@import type { UserContent } from "ai";import type { ChannelAdapter } from "#channel/adapter.js";+import {+ createChannelDeliveryMetadata,+ type ChannelDeliverySource,+} from "#channel/delivery-metadata.js";import type { SendPayload } from "#channel/routes.js";import { normalizeSendInput, serializeUrlFilePartsInMessage } from "#channel/send-input.js";import { createSession, sessionCallbackToTurnCaller, type Session } from "#channel/session.js";@@ -67,11 +71,11 @@ export function createChannelAddress<TState = undefined>(input: {readonly adapter: ChannelAdapter<any>;readonly channelName: string;readonly continuationToken: string;- readonly metadata?: { readonly requestId?: string };+ readonly metadata?: ChannelDeliverySource;readonly runtime: Runtime;readonly turnPolicy?: TurnPolicy;}): ChannelAddress<TState> {- const metadata = input.metadata ?? {};+ const metadata: Partial<ChannelDeliverySource> = input.metadata ?? {};const namespacedToken = `${input.channelName}:${input.continuationToken}`;if (isReservedSessionCommandToken(namespacedToken)) {throw new Error(`Channel address "${namespacedToken}" uses eve's reserved session namespace.`);@@ -80,10 +84,15 @@ export function createChannelAddress<TState = undefined>(input: {return {continuationToken: input.continuationToken,async deliver(sendInput, options) {+ const delivery =+ metadata.channelKind !== undefined && metadata.channelName !== undefined+ ? createChannelDeliveryMetadata(metadata as ChannelDeliverySource)+ : undefined;const payload = normalizeSendInput(sendInput);const caller = sessionCallbackToTurnCaller(options.callback);const commandWithoutCaller = {auth: options.auth,+ delivery,kind: "send" as const,payload: {...payload,@@ -133,6 +142,7 @@ export function createChannelAddress<TState = undefined>(input: {callback: options.callback,channelName: input.channelName,continuationToken: namespacedToken,+ delivery,initiatorAuth: options.initiatorAuth,input: {context: payload.context,@@ -205,7 +215,7 @@ export function createChannelAddress<TState = undefined>(input: {export function createChannelAddressFn<TState = undefined>(input: {readonly adapter: ChannelAdapter<any>;readonly channelName: string;- readonly metadata?: { readonly requestId?: string };+ readonly metadata?: ChannelDeliverySource;readonly runtime: Runtime;readonly turnPolicy?: TurnPolicy;}): ChannelAddressFn<TState> {diff --git a/packages/eve/src/channel/channel-operations.ts b/packages/eve/src/channel/channel-operations.tsindex 132e86460..d2a25ef19 100644--- a/packages/eve/src/channel/channel-operations.ts+++ b/packages/eve/src/channel/channel-operations.ts@@ -1,6 +1,7 @@import type { UserContent } from "ai";import type { ChannelAdapter } from "#channel/adapter.js";+import type { ChannelDeliverySource } from "#channel/delivery-metadata.js";import {createChannelAddressFn,type ChannelAddressDeliveryOptions,@@ -91,7 +92,7 @@ export interface InternalChannelSource<TState = undefined> extends ChannelSourceexport function createChannelOperations<TState = undefined>(input: {readonly adapter: ChannelAdapter<any>;readonly channelName: string;- readonly metadata?: { readonly requestId?: string };+ readonly metadata?: ChannelDeliverySource;readonly runtime: Runtime;readonly turnPolicy?: TurnPolicy;}): ChannelReceiveContext<TState> {diff --git a/packages/eve/src/channel/delivery-metadata.ts b/packages/eve/src/channel/delivery-metadata.tsnew file mode 100644index 000000000..815a90ef1--- /dev/null+++ b/packages/eve/src/channel/delivery-metadata.ts@@ -0,0 +1,15 @@+import type { ChannelDeliveryMetadata, SessionTraceContext } from "#channel/types.js";++export interface ChannelDeliverySource {+ readonly channelKind: string;+ readonly channelName: string;+ readonly requestId?: string;+ readonly requestTraceContext?: SessionTraceContext;+}++/** Mints the opaque identity for one inbound channel operation. */+export function createChannelDeliveryMetadata(+ source: ChannelDeliverySource,+): ChannelDeliveryMetadata {+ return { ...source, deliveryId: crypto.randomUUID() };+}diff --git a/packages/eve/src/channel/session.ts b/packages/eve/src/channel/session.tsindex ac46e7d13..807d4d2f1 100644--- a/packages/eve/src/channel/session.ts+++ b/packages/eve/src/channel/session.ts@@ -1,4 +1,8 @@import type { ContextAccessor } from "#context/key.js";+import {+ createChannelDeliveryMetadata,+ type ChannelDeliverySource,+} from "#channel/delivery-metadata.js";import type { MessageStreamEvent } from "#protocol/message.js";import type { UserContent } from "ai";import type {@@ -79,11 +83,12 @@ export interface SessionHandle {export function createSession(id: string,runtime: Runtime,- metadata: { readonly requestId?: string; readonly turnPolicy?: TurnPolicy } = {},+ metadata: Partial<ChannelDeliverySource> & { readonly turnPolicy?: TurnPolicy } = {},): Session {return {id,async send(message, options) {+ const delivery = createDelivery(metadata);const caller = sessionCallbackToTurnCaller(options.callback);const payload: {context?: readonly string[];@@ -94,6 +99,7 @@ export function createSession(if (options.outputSchema !== undefined) payload.outputSchema = options.outputSchema;const commandWithoutCaller = {auth: options.auth,+ delivery,kind: "send" as const,payload,requestId: metadata.requestId,@@ -109,6 +115,7 @@ export function createSession(throw new Error("respond() requires at least one input response.");}const caller = sessionCallbackToTurnCaller(options.callback);+ const delivery = createDelivery(metadata);const payload: {context?: readonly string[];inputResponses: readonly InputResponse[];@@ -118,6 +125,7 @@ export function createSession(if (options.outputSchema !== undefined) payload.outputSchema = options.outputSchema;const commandWithoutCaller = {auth: options.auth,+ delivery,kind: "send" as const,payload,requestId: metadata.requestId,@@ -157,11 +165,19 @@ export function createSession(/** Builds an I/O-free factory for fixed session-ID handles. */export function createAttachSessionFn(runtime: Runtime,- metadata: { readonly requestId?: string; readonly turnPolicy?: TurnPolicy } = {},+ metadata: Partial<ChannelDeliverySource> & { readonly turnPolicy?: TurnPolicy } = {},): (sessionId: string) => Session {return (sessionId) => createSession(sessionId, runtime, metadata);}+function createDelivery(+ metadata: Partial<ChannelDeliverySource>,+): ReturnType<typeof createChannelDeliveryMetadata> | undefined {+ return metadata.channelKind !== undefined && metadata.channelName !== undefined+ ? createChannelDeliveryMetadata(metadata as ChannelDeliverySource)+ : undefined;+}+/*** Builds a live {@link SessionHandle} backed by the active context* accessor. Read-only fields resolve through getters so they reflectdiff --git a/packages/eve/src/channel/types.ts b/packages/eve/src/channel/types.tsindex e3a5c6bfb..fa49abd6e 100644--- a/packages/eve/src/channel/types.ts+++ b/packages/eve/src/channel/types.ts@@ -85,6 +85,20 @@ export interface SessionTraceContext {readonly traceId: string;}+/** Framework-owned identity for one inbound channel operation. */+export interface ChannelDeliveryMetadata {+ readonly channelKind: string;+ readonly channelName: string;+ readonly deliveryId: string;+ readonly requestId?: string;+ readonly requestTraceContext?: SessionTraceContext;+}++/** Associates delivery metadata with one payload in a durable envelope. */+export interface ChannelDeliveryMetadataEntry extends ChannelDeliveryMetadata {+ readonly payloadIndex: number;+}+// ---------------------------------------------------------------------------// Auth// ---------------------------------------------------------------------------@@ -159,6 +173,7 @@ export type SessionCommand =readonly caller?: TurnCaller;readonly kind: "send";readonly payload: DeliverPayload;+ readonly delivery?: ChannelDeliveryMetadata;readonly requestId?: string;readonly turnPolicy?: TurnPolicy;}@@ -211,6 +226,8 @@ export interface DeliverHookPayload {readonly auth?: SessionAuthContext | null;/** Delegated caller waiting for this turn's settled result. */readonly caller?: TurnCaller;+ /** Additive durable metadata. Absent on envelopes written by older deployments. */+ readonly deliveryMetadata?: readonly ChannelDeliveryMetadataEntry[];/** Inbound channel request id used only for workflow attributes. */readonly requestId?: string;readonly kind: "deliver";@@ -376,6 +393,8 @@ export interface RunInput {*/readonly channelName?: string;readonly channelMetadata?: ChannelInstrumentationProjection;+ /** Inbound channel operation that created this session. */+ readonly delivery?: ChannelDeliveryMetadata;/*** Authenticated caller principal for this session. `null` means the* request was accepted with no credentials.diff --git a/packages/eve/src/context/keys.ts b/packages/eve/src/context/keys.tsindex ef1c9d35a..52ecbd1c2 100644--- a/packages/eve/src/context/keys.ts+++ b/packages/eve/src/context/keys.ts@@ -9,6 +9,7 @@ import type { LanguageModel, SystemModelMessage } from "ai";import type { JsonObject } from "#shared/json.js";import type {ChannelInstrumentationProjection,+ ChannelDeliveryMetadata,SessionAuthContext,SessionCallback,SessionCapabilities,@@ -17,6 +18,7 @@ import type {SessionTurn,} from "#channel/types.js";import { ContextKey } from "#context/key.js";+import type { InstrumentationChannelDeliveryRef } from "#harness/instrumentation-lifecycle.js";import type { HarnessToolDefinition } from "#harness/execute-tool.js";import type { DynamicSubagentAgentConfig } from "#runtime/subagents/dynamic-agent-config.js";import type { DynamicRemoteAgentConfig } from "#runtime/subagents/dynamic-remote-agent-config.js";@@ -67,6 +69,18 @@ export const InitiatorAuthKey = new ContextKey<SessionAuthContext | null>("eve.iexport const SessionIdKey = new ContextKey<string>("eve.sessionId");export const ContinuationTokenKey = new ContextKey<string>("eve.continuationToken");export const ChannelRequestIdKey = new ContextKey<string>("eve.channelRequestId");+export const ChannelDeliveryKey = new ContextKey<ChannelDeliveryMetadata>("eve.channelDelivery");+export interface ActiveChannelDelivery {+ readonly agentName?: string;+ readonly delivery: InstrumentationChannelDeliveryRef;+ readonly rootSessionId: string;+ readonly sequence: number;+ readonly sessionId: string;+ readonly turnId: string;+}+export const ActiveChannelDeliveriesKey = new ContextKey<readonly ActiveChannelDelivery[]>(+ "eve.activeChannelDeliveries",+);export const ChannelInstrumentationKey = new ContextKey<ChannelInstrumentationProjection>("eve.channelInstrumentation",);diff --git a/packages/eve/src/execution/dispatch-turn-step.ts b/packages/eve/src/execution/dispatch-turn-step.tsnew file mode 100644index 000000000..1cd87aeed--- /dev/null+++ b/packages/eve/src/execution/dispatch-turn-step.ts@@ -0,0 +1,31 @@+import {+ createTurnWorkflowInput,+ type TurnWorkflowDispatchInput,+} from "#execution/durable-session-migrations/turn-workflow.js";+import { buildTurnAttributes, readRootSessionId } from "#execution/eve-workflow-attributes.js";+import { startWorkflowPreferLatest, turnWorkflowReference } from "#execution/workflow-runtime.js";+import { normalizeEveAttributes } from "#runtime/attributes/normalize.js";++/** Starts a per-turn child workflow for the current driver session. */+export async function dispatchTurnStep(+ input: TurnWorkflowDispatchInput,+): Promise<{ readonly runId: string }> {+ "use step";++ const run = await startWorkflowPreferLatest(+ turnWorkflowReference,+ [createTurnWorkflowInput(input)],+ {+ allowReservedAttributes: true,+ attributes: normalizeEveAttributes(+ buildTurnAttributes({+ parentSessionId: input.sessionState.sessionId,+ requestId: input.delivery.kind === "deliver" ? input.delivery.requestId : undefined,+ rootSessionId: readRootSessionId(input.serializedContext) ?? input.sessionState.sessionId,+ }),+ ),+ },+ );++ return { runId: run.runId };+}diff --git a/packages/eve/src/execution/parked-delivery-wait.test.ts b/packages/eve/src/execution/parked-delivery-wait.test.tsindex a4c369c83..4e5541c7b 100644--- a/packages/eve/src/execution/parked-delivery-wait.test.ts+++ b/packages/eve/src/execution/parked-delivery-wait.test.ts@@ -1,4 +1,4 @@-import { afterEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import type { DeliverHookPayload } from "#channel/types.js";import { nextTurnDelivery } from "#execution/parked-delivery-wait.js";@@ -90,6 +90,13 @@ function messageRead(message: string): ScriptedRead {};}+function runtimeActionResultRead(): ScriptedRead {+ return {+ result: { done: false, value: { kind: "runtime-action-result", results: [] } },+ source: "session",+ };+}+// Routing never runs in these tests: scripted reads stop at authorization// instructions or exhaust before any deliver-kind turn payload.const sessionState = { sessionId: "ses-parked-wait" } as DurableSessionState;@@ -106,6 +113,14 @@ function waitInput(inbox: SessionCommandInbox): Parameters<typeof nextTurnDelive}describe("nextTurnDelivery", () => {+ beforeEach(() => {+ vi.mocked(routeDeliverToChildren).mockImplementation(async ({ delivery, sessionState }) => ({+ kind: "continue",+ remainder: delivery,+ sessionState,+ }));+ });+afterEach(() => {vi.mocked(routeDeliverToChildren).mockReset();});@@ -134,6 +149,20 @@ describe("nextTurnDelivery", () => {expect(inbox.windowTransitions).toEqual([true, false]);});+ it("ignores child results already owned by the runtime-action collector", async () => {+ const inbox = createMockInbox([runtimeActionResultRead(), messageRead("follow up")]);++ const next = await nextTurnDelivery({+ ...waitInput(inbox),+ awaitAuthorizationCallbacks: false,+ });++ expect(next).toMatchObject({+ kind: "turn",+ remainder: { kind: "deliver", payloads: [{ message: "follow up" }] },+ });+ });+it("does not let buffered deliveries bypass a ready authorization callback", async () => {const inbox = createMockInbox([authorizationRead()], true);const bufferedDeliveries: DeliverHookPayload[] = [@@ -195,7 +224,7 @@ describe("nextTurnDelivery", () => {}).mockResolvedValueOnce({kind: "continue",- remainder: { message: "parent turn" },+ remainder: { kind: "deliver", payloads: [{ message: "parent turn" }] },sessionState: retiredState,});diff --git a/packages/eve/src/execution/parked-delivery-wait.ts b/packages/eve/src/execution/parked-delivery-wait.tsindex 49086143f..da9afb9f9 100644--- a/packages/eve/src/execution/parked-delivery-wait.ts+++ b/packages/eve/src/execution/parked-delivery-wait.ts@@ -37,7 +37,7 @@ export type NextTurnInstruction =| {readonly kind: "turn";readonly deliver: DeliverHookPayload;- readonly remainder: DeliverPayload;+ readonly remainder: DeliverHookPayload;readonly sessionState: DurableSessionState;};@@ -107,9 +107,8 @@ async function awaitNextTurnDelivery(input: {}const routed = await routeDeliverToChildren({- auth: deliver.auth,+ delivery: deliver,parentWritable: input.driverWritable,- payloads: deliver.payloads,sessionState,});sessionState = routed.sessionState;@@ -184,6 +183,12 @@ async function waitForNextSessionAction(input: {continue;}+ // Child results also arrive through this inbox, but the runtime-action+ // collector owns them. They are not channel deliveries.+ if (first.value.kind === "runtime-action-result") {+ continue;+ }+if (first.value.kind === "deliver") {if (input.deferDeliveries === true) {input.bufferedDeliveries.push(first.value);diff --git a/packages/eve/src/execution/route-child-delivery.ts b/packages/eve/src/execution/route-child-delivery.tsindex 9a5d3901c..5262f6973 100644--- a/packages/eve/src/execution/route-child-delivery.ts+++ b/packages/eve/src/execution/route-child-delivery.ts@@ -1,7 +1,9 @@-import type { DeliverPayload, SessionAuthContext } from "#channel/types.js";-import { coalesceDeliverPayloads } from "#execution/deliver-payloads.js";+import type { DeliverHookPayload } from "#channel/types.js";import type { DurableSessionState } from "#execution/durable-session-store.js";-import { routeProxiedDeliverStep, type RoutedDeliverResult } from "#execution/workflow-steps.js";+import {+ routeProxiedDeliverStep,+ type RoutedDeliverResult,+} from "#execution/route-proxied-deliver-step.js";/*** Coalesces inbound deliver payloads and routes any descendant-bound input@@ -14,20 +16,17 @@ import { routeProxiedDeliverStep, type RoutedDeliverResult } from "#execution/wo* re-export plain helpers into a workflow body).*/export async function routeDeliverToChildren(input: {- readonly auth?: SessionAuthContext | null;+ readonly delivery: DeliverHookPayload;readonly parentWritable: WritableStream<Uint8Array>;- readonly payloads: readonly DeliverPayload[];readonly sessionState: DurableSessionState;}): Promise<RoutedDeliverResult> {- const payload = coalesceDeliverPayloads(input.payloads);if (!input.sessionState.hasProxyInputRequests) {- return { kind: "continue", remainder: payload, sessionState: input.sessionState };+ return { kind: "continue", remainder: input.delivery, sessionState: input.sessionState };}return await routeProxiedDeliverStep({- auth: input.auth,+ delivery: input.delivery,parentWritable: input.parentWritable,- payload,sessionState: input.sessionState,});}diff --git a/packages/eve/src/execution/route-proxied-deliver-step.ts b/packages/eve/src/execution/route-proxied-deliver-step.tsnew file mode 100644index 000000000..50ff04296--- /dev/null+++ b/packages/eve/src/execution/route-proxied-deliver-step.ts@@ -0,0 +1,140 @@+import type { DeliverHookPayload, DeliverPayload, SessionAuthContext } from "#channel/types.js";+import { coalesceDeliverPayloads } from "#execution/deliver-payloads.js";+import {+ type DurableSessionState,+ readDurableSession,+ replaceDurableSessionSnapshot,+} from "#execution/durable-session-store.js";+import { routeDeliverPayload } from "#execution/subagent-hitl-proxy.js";+import { retireProxyInputRequests } from "#harness/proxy-input-requests.js";+import { resumeHook } from "#internal/workflow/runtime.js";++export type RoutedDeliverResult =+ | { readonly kind: "cancel-turn"; readonly sessionState: DurableSessionState }+ | {+ readonly kind: "continue";+ readonly remainder: DeliverHookPayload | undefined;+ readonly sessionState: DurableSessionState;+ };++type LegacyRoutedDeliverResult =+ | { readonly kind: "cancel-turn"; readonly sessionState: DurableSessionState }+ | {+ readonly kind: "continue";+ readonly remainder: DeliverPayload | undefined;+ readonly sessionState: DurableSessionState;+ };++/**+ * Splits an inbound deliver payload into parent-local and proxied-child buckets.+ * Successfully forwarded request IDs are retired from the returned snapshot.+ */+export function routeProxiedDeliverStep(input: {+ readonly delivery: DeliverHookPayload;+ readonly parentWritable: WritableStream<Uint8Array>;+ readonly sessionState: DurableSessionState;+}): Promise<RoutedDeliverResult>;+export function routeProxiedDeliverStep(input: {+ readonly auth?: SessionAuthContext | null;+ readonly parentWritable: WritableStream<Uint8Array>;+ readonly payload: DeliverPayload;+ readonly sessionState: DurableSessionState;+}): Promise<LegacyRoutedDeliverResult>;+export async function routeProxiedDeliverStep(+ input:+ | {+ readonly delivery: DeliverHookPayload;+ readonly parentWritable: WritableStream<Uint8Array>;+ readonly sessionState: DurableSessionState;+ }+ | {+ readonly auth?: SessionAuthContext | null;+ readonly parentWritable: WritableStream<Uint8Array>;+ readonly payload: DeliverPayload;+ readonly sessionState: DurableSessionState;+ },+): Promise<LegacyRoutedDeliverResult | RoutedDeliverResult> {+ "use step";++ let durableSession = await readDurableSession(input.sessionState);+ const legacyInput = !("delivery" in input);+ const sourceDelivery: DeliverHookPayload =+ "delivery" in input+ ? input.delivery+ : { auth: input.auth, kind: "deliver", payloads: [input.payload] };+ const metadataByPayload = new Map(+ (sourceDelivery.deliveryMetadata ?? []).map((entry) => [entry.payloadIndex, entry]),+ );+ const parentPayloads: DeliverPayload[] = [];+ const parentMetadata: NonNullable<DeliverHookPayload["deliveryMetadata"]>[number][] = [];+ const children = new Map<+ string,+ {+ metadata: NonNullable<DeliverHookPayload["deliveryMetadata"]>[number][];+ payloads: DeliverPayload[];+ retireRequestIds: string[];+ }+ >();+ let parentAction: { readonly kind: "cancel-turn" } | undefined;++ for (const [payloadIndex, payload] of sourceDelivery.payloads.entries()) {+ const routed = routeDeliverPayload({ payload, state: durableSession.state });+ parentAction ??= routed.parentAction;+ const metadata = metadataByPayload.get(payloadIndex);++ if (routed.forSelf !== undefined) {+ const nextIndex = parentPayloads.length;+ parentPayloads.push(routed.forSelf);+ if (metadata !== undefined) parentMetadata.push({ ...metadata, payloadIndex: nextIndex });+ }++ for (const [childIndex, forChild] of routed.forChildren.entries()) {+ const child = children.get(forChild.childContinuationToken) ?? {+ metadata: [],+ payloads: [],+ retireRequestIds: [],+ };+ const nextIndex = child.payloads.length;+ child.payloads.push(forChild.payload);+ child.retireRequestIds.push(...forChild.retireRequestIds);+ if (metadata !== undefined && routed.forSelf === undefined && childIndex === 0) {+ child.metadata.push({ ...metadata, payloadIndex: nextIndex });+ }+ children.set(forChild.childContinuationToken, child);+ }+ }++ for (const [childContinuationToken, child] of children) {+ const delivery: DeliverHookPayload = {+ auth: sourceDelivery.auth,+ caller: sourceDelivery.caller,+ deliveryMetadata: child.metadata.length === 0 ? undefined : child.metadata,+ kind: "deliver",+ payloads: child.payloads,+ requestId: sourceDelivery.requestId,+ };+ await resumeHook(childContinuationToken, {+ ...delivery,+ payload: coalesceDeliverPayloads(child.payloads),+ });+ durableSession = retireProxyInputRequests(durableSession, child.retireRequestIds);+ }++ const sessionState =+ children.size === 0+ ? input.sessionState+ : replaceDurableSessionSnapshot({ session: durableSession, state: input.sessionState });++ if (parentAction !== undefined) return { ...parentAction, sessionState };+ const remainder =+ parentPayloads.length === 0+ ? undefined+ : legacyInput+ ? coalesceDeliverPayloads(parentPayloads)+ : {+ ...sourceDelivery,+ deliveryMetadata: parentMetadata.length === 0 ? undefined : parentMetadata,+ payloads: parentPayloads,+ };+ return { kind: "continue", remainder, sessionState };+}diff --git a/packages/eve/src/execution/runtime-context.ts b/packages/eve/src/execution/runtime-context.tsindex 7f625b9fc..ebae5ec9b 100644--- a/packages/eve/src/execution/runtime-context.ts+++ b/packages/eve/src/execution/runtime-context.ts@@ -5,6 +5,7 @@ import {AuthKey,CapabilitiesKey,ChannelInstrumentationKey,+ ChannelDeliveryKey,ChannelRequestIdKey,ContinuationTokenKey,DynamicSubagentAgentConfigKey,@@ -60,6 +61,10 @@ export function buildRunContext(input: {ctx.set(ChannelRequestIdKey, run.requestId);}+ if (run.delivery !== undefined) {+ ctx.set(ChannelDeliveryKey, run.delivery);+ }+if (run.callback !== undefined) {ctx.set(SessionCallbackKey, run.callback);}diff --git a/packages/eve/src/execution/session-command-inbox.integration.test.ts b/packages/eve/src/execution/session-command-inbox.integration.test.tsindex 8fcf473e8..fb33206d4 100644--- a/packages/eve/src/execution/session-command-inbox.integration.test.ts+++ b/packages/eve/src/execution/session-command-inbox.integration.test.ts@@ -22,7 +22,15 @@ describe("session command inbox integration", () => {await expect(runtime.dispatchContinuation({- command: { kind: "send", payload: { message: "legacy-compatible" } },+ command: {+ delivery: {+ channelKind: "channel:http",+ channelName: "http",+ deliveryId: "legacy-delivery",+ },+ kind: "send",+ payload: { message: "legacy-compatible" },+ },continuationToken: token,}),).resolves.toEqual({ sessionId: run.runId, status: "accepted" });@@ -45,7 +53,15 @@ describe("session command inbox integration", () => {await expect(runtime.dispatchContinuation({- command: { kind: "send", payload: { message: "mid-cohort-compatible" } },+ command: {+ delivery: {+ channelKind: "channel:http",+ channelName: "http",+ deliveryId: "mid-cohort-delivery",+ },+ kind: "send",+ payload: { message: "mid-cohort-compatible" },+ },continuationToken: token,}),).resolves.toEqual({ sessionId: run.runId, status: "accepted" });diff --git a/packages/eve/src/execution/session-command-inbox.ts b/packages/eve/src/execution/session-command-inbox.tsindex afc436f21..6754c625f 100644--- a/packages/eve/src/execution/session-command-inbox.ts+++ b/packages/eve/src/execution/session-command-inbox.ts@@ -2,6 +2,7 @@ import { createHook, type Hook } from "#compiled/@workflow/core/index.js";import type {DeliverHookPayload,+ RuntimeActionResultHookPayload,SessionCommand,SessionTimeoutHookPayload,} from "#channel/types.js";@@ -16,7 +17,11 @@ import { claimHookOwnership, disposeHook } from "#execution/hook-ownership.js";* sessions are bounded by the 30-day default timeout, so the decode can be* dropped once runs created on those versions have aged out.*/-export type SessionInboxPayload = DeliverHookPayload | SessionCommand | SessionTimeoutHookPayload;+export type SessionInboxPayload =+ | DeliverHookPayload+ | RuntimeActionResultHookPayload+ | SessionCommand+ | SessionTimeoutHookPayload;interface HookRead {readonly order: number;diff --git a/packages/eve/src/execution/session-command-wire.test.ts b/packages/eve/src/execution/session-command-wire.test.tsindex a2cff61b2..bee53c8ad 100644--- a/packages/eve/src/execution/session-command-wire.test.ts+++ b/packages/eve/src/execution/session-command-wire.test.ts@@ -33,4 +33,18 @@ describe("sendCommandToDelivery", () => {// `payloads`. Both views must reference the same delivery.expect(wire.payloads[0]).toBe(wire.payload);});++ it("preserves delivery identity on the durable envelope", () => {+ const delivery = {+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-1",+ requestId: "request-1",+ };++ expect(+ sendCommandToDelivery({ delivery, kind: "send", payload: { message: "hello" } })+ .deliveryMetadata,+ ).toEqual([{ ...delivery, payloadIndex: 0 }]);+ });});diff --git a/packages/eve/src/execution/session-command-wire.ts b/packages/eve/src/execution/session-command-wire.tsindex 813f0dd23..7c44e08bb 100644--- a/packages/eve/src/execution/session-command-wire.ts+++ b/packages/eve/src/execution/session-command-wire.ts@@ -27,6 +27,8 @@ export function sendCommandToDelivery(return {auth: command.auth,caller: command.caller,+ deliveryMetadata:+ command.delivery === undefined ? undefined : [{ ...command.delivery, payloadIndex: 0 }],kind: "deliver",payload: command.payload,payloads: [command.payload],diff --git a/packages/eve/src/execution/settle-cancelled-turn-step.ts b/packages/eve/src/execution/settle-cancelled-turn-step.tsindex f5efab0b0..1c75ab6d5 100644--- a/packages/eve/src/execution/settle-cancelled-turn-step.ts+++ b/packages/eve/src/execution/settle-cancelled-turn-step.ts@@ -3,6 +3,7 @@ import { callAdapterEventHandler } from "#channel/adapter.js";import { dispatchStreamEventHooks } from "#context/hook-lifecycle.js";import { withContextScope } from "#context/run-step.js";import { deserializeContext, serializeContext } from "#context/serialize.js";+import { ChannelInstrumentationKey } from "#context/keys.js";import { setChannelContext } from "#execution/channel-context.js";import {createDurableSessionState,@@ -103,6 +104,7 @@ export async function settleCancelledTurnStep(input: {const emit =createInstrumentationHandleEvent({agentName: bundle.turnAgent.id,+ channelKind: ctx.get(ChannelInstrumentationKey)?.kind,handleEvent: baseEmit,hooks: instrumentation?.hooks,sessionId: session.sessionId,diff --git a/packages/eve/src/execution/turn-control-receiver.ts b/packages/eve/src/execution/turn-control-receiver.tsindex 6c1f90569..a264dec52 100644--- a/packages/eve/src/execution/turn-control-receiver.ts+++ b/packages/eve/src/execution/turn-control-receiver.ts@@ -88,6 +88,9 @@ export class TurnControlReceiver {this.bufferedSessionControls.push("expired");return undefined;}+ if (command.kind === "runtime-action-result") {+ return undefined;+ }if (command.kind === "cancel") {await forwardTurnCancellationStep({payload: command.turnId === undefined ? {} : { turnId: command.turnId },diff --git a/packages/eve/src/execution/turn-dispatch.test.ts b/packages/eve/src/execution/turn-dispatch.test.tsindex 52a69a2d6..fd745a040 100644--- a/packages/eve/src/execution/turn-dispatch.test.ts+++ b/packages/eve/src/execution/turn-dispatch.test.ts@@ -13,7 +13,7 @@ vi.mock("#compiled/@workflow/core/index.js", () => ({createHook: (...args: unknown[]) => createHookMock(...args),}));-vi.mock("./workflow-steps.js", () => ({+vi.mock("./dispatch-turn-step.js", () => ({dispatchTurnStep: vi.fn(async () => ({ runId: "turn-run" })),}));diff --git a/packages/eve/src/execution/turn-dispatch.ts b/packages/eve/src/execution/turn-dispatch.tsindex 3f1c28615..7313d6447 100644--- a/packages/eve/src/execution/turn-dispatch.ts+++ b/packages/eve/src/execution/turn-dispatch.ts@@ -1,8 +1,8 @@import type { DeliverHookPayload, HookPayload, SessionCapabilities } from "#channel/types.js";+import { dispatchTurnStep } from "#execution/dispatch-turn-step.js";import { TurnControlReceiver } from "#execution/turn-control-receiver.js";import type { DurableSessionState } from "#execution/durable-session-store.js";import type { SessionCommandInbox } from "#execution/session-command-inbox.js";-import { dispatchTurnStep } from "#execution/workflow-steps.js";import type { TurnDriverAction } from "#execution/turn-control-receiver.js";import type { RunMode } from "#shared/run-mode.js";diff --git a/packages/eve/src/execution/turn-workflow.test.ts b/packages/eve/src/execution/turn-workflow.test.tsindex e4256bcb6..37b666641 100644--- a/packages/eve/src/execution/turn-workflow.test.ts+++ b/packages/eve/src/execution/turn-workflow.test.ts@@ -839,7 +839,10 @@ describe("turnWorkflow", () => {});expect(routeDeliverToChildren).toHaveBeenCalledWith(expect.objectContaining({- payloads: [{ inputResponses: [{ optionId: "approve", requestId: "approval-1" }] }],+ delivery: {+ kind: "deliver",+ payloads: [{ inputResponses: [{ optionId: "approve", requestId: "approval-1" }] }],+ },sessionState: proxyState,}),);diff --git a/packages/eve/src/execution/turn-workflow.ts b/packages/eve/src/execution/turn-workflow.tsindex 313902acf..d548f34d5 100644--- a/packages/eve/src/execution/turn-workflow.ts+++ b/packages/eve/src/execution/turn-workflow.ts@@ -376,9 +376,8 @@ async function waitForRuntimeActionResults(input: {pendingDeliveryRequest = undefined;const routed = await routeDeliverToChildren({- auth: value.delivery.auth,+ delivery: value.delivery,parentWritable: input.cursor.parentWritable,- payloads: value.delivery.payloads,sessionState: input.cursor.sessionState,});await input.cursor.adopt(routed);@@ -386,7 +385,7 @@ async function waitForRuntimeActionResults(input: {return routed.kind;}if (routed.remainder !== undefined) {- input.bufferedDeliveries.push({ ...value.delivery, payloads: [routed.remainder] });+ input.bufferedDeliveries.push(routed.remainder);}}}diff --git a/packages/eve/src/execution/workflow-entry.test.ts b/packages/eve/src/execution/workflow-entry.test.tsindex e2b12ab8f..4161c28c2 100644--- a/packages/eve/src/execution/workflow-entry.test.ts+++ b/packages/eve/src/execution/workflow-entry.test.ts@@ -11,6 +11,7 @@ import {resolveInitialTurnCallerStep,} from "#execution/delegated-parent-notification.js";import type { DurableSessionState } from "#execution/durable-session-store.js";+import { dispatchTurnStep } from "#execution/dispatch-turn-step.js";import { fireSessionCallbackStep } from "#execution/session-callback-step.js";import { emitTerminalSessionCompletionStep } from "#execution/terminal-session-completion-step.js";import {@@ -22,7 +23,6 @@ import type { TurnControlPayload } from "#execution/turn-control-protocol.js";import { workflowEntry } from "#execution/workflow-entry.js";import { routeDeliverToChildren } from "#execution/route-child-delivery.js";import { settleCancelledTurnStep } from "#execution/settle-cancelled-turn-step.js";-import { dispatchTurnStep } from "#execution/workflow-steps.js";import { emitTerminalSessionFailureStep } from "#execution/terminal-session-failure-step.js";import type { SessionInboxPayload } from "#execution/session-command-inbox.js";import { sessionCommandHookToken } from "#execution/session-command-token.js";@@ -58,9 +58,9 @@ vi.mock("./create-session-step.js", () => ({}));vi.mock("./route-child-delivery.js", () => ({- routeDeliverToChildren: vi.fn().mockImplementation(async ({ payloads, sessionState }) => ({+ routeDeliverToChildren: vi.fn().mockImplementation(async ({ delivery, sessionState }) => ({kind: "continue",- remainder: payloads[0],+ remainder: delivery,sessionState,})),}));@@ -75,7 +75,7 @@ vi.mock("./terminate-child-sessions-step.js", () => ({terminateChildSessionsStep: vi.fn().mockResolvedValue(undefined),}));-vi.mock("./workflow-steps.js", () => ({+vi.mock("./dispatch-turn-step.js", () => ({dispatchTurnStep: vi.fn().mockImplementation(async () => ({ runId: "turn-run" })),}));@@ -843,7 +843,7 @@ describe("workflowEntry", () => {serializedContext: createSerializedContext(),});- expect(vi.mocked(dispatchTurnStep).mock.calls[1]?.[0].delivery).toEqual({+ expect(vi.mocked(dispatchTurnStep).mock.calls[1]?.[0].delivery).toMatchObject({auth: undefined,requestId: "req_followup",kind: "deliver",@@ -976,7 +976,7 @@ describe("workflowEntry", () => {expect(result).toEqual({ output: "after delivery" });expect(dispatchTurnStep).toHaveBeenCalledTimes(2);- expect(vi.mocked(dispatchTurnStep).mock.calls[1]?.[0].delivery).toEqual({+ expect(vi.mocked(dispatchTurnStep).mock.calls[1]?.[0].delivery).toMatchObject({auth: undefined,kind: "deliver",payloads: [{ message: "not for the child" }],@@ -1410,7 +1410,7 @@ describe("workflowEntry", () => {expect(result).toEqual({ output: "" });expect(nonTurnHookTokens()).toEqual(["slack:C01:", "slack:C01:1800000000.123456"]);- expect(vi.mocked(dispatchTurnStep).mock.calls[1]?.[0].delivery).toEqual({+ expect(vi.mocked(dispatchTurnStep).mock.calls[1]?.[0].delivery).toMatchObject({kind: "deliver",payloads: [{ message: "follow up" }],});diff --git a/packages/eve/src/execution/workflow-entry.ts b/packages/eve/src/execution/workflow-entry.tsindex bdf4fb39b..ab02698f9 100644--- a/packages/eve/src/execution/workflow-entry.ts+++ b/packages/eve/src/execution/workflow-entry.ts@@ -182,6 +182,17 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflowcapabilities,driverWritable,initialInput: {+ deliveryMetadata:+ input.serializedContext["eve.channelDelivery"] === undefined+ ? undefined+ : [+ {+ ...(input.serializedContext["eve.channelDelivery"] as NonNullable<+ RunInput["delivery"]+ >),+ payloadIndex: 0,+ },+ ],kind: "deliver",payloads: [{@@ -559,10 +570,8 @@ async function runDriverLoop(input: {}action = await runTurn({delivery: {- auth: next.deliver.auth,+ ...next.remainder,kind: "deliver",- payloads: [next.remainder],- requestId: next.deliver.requestId,},serializedContext: action.serializedContext,sessionState: action.sessionState,diff --git a/packages/eve/src/execution/workflow-steps.test.ts b/packages/eve/src/execution/workflow-steps.test.tsindex 053a9b0e6..6ab61c754 100644--- a/packages/eve/src/execution/workflow-steps.test.ts+++ b/packages/eve/src/execution/workflow-steps.test.ts@@ -33,18 +33,15 @@ import {readDurableSession,} from "#execution/durable-session-store.js";import { createTurnWorkflowInput } from "#execution/durable-session-migrations/turn-workflow.js";+import { dispatchTurnStep } from "#execution/dispatch-turn-step.js";import { projectToDurableSession } from "#execution/session.js";import { buildRuntimeIdentity, createExecutionNodeStep } from "#execution/node-step.js";import { defineTool } from "#public/definitions/tool.js";import { dispatchRuntimeActionsStep } from "#execution/dispatch-runtime-actions-step.js";import { runProxySubagentEventStep } from "#execution/subagent-event-proxy-step.js";import { emitTerminalSessionFailureStep } from "#execution/terminal-session-failure-step.js";-import {- dispatchTurnStep,- routeProxiedDeliverStep,- resolveEffectiveOutputSchema,- turnStep,-} from "#execution/workflow-steps.js";+import { routeProxiedDeliverStep } from "#execution/route-proxied-deliver-step.js";+import { resolveEffectiveOutputSchema, turnStep } from "#execution/workflow-steps.js";import {LATEST_DEPLOYMENT_UNSUPPORTED_MESSAGE,turnWorkflowReference,@@ -343,6 +340,40 @@ describe("routeProxiedDeliverStep", () => {expect(resumeHookMock.mock.calls.map(([token]) => token)).toEqual(["child-a", "child-b"]);});++ it("preserves metadata when the descendant wholly owns the delivery", async () => {+ const session = upsertProxyInputRequests({+ entries: [["request-1", { childContinuationToken: "child-token", kind: "tool-approval" }]],+ forChildContinuationToken: "child-token",+ session: createStubSession({ sessionId: "parent-session" }),+ });+ installSessionStoreMocks([session]);+ const metadata = {+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-1",+ payloadIndex: 0,+ requestId: "request-http-1",+ };++ await routeProxiedDeliverStep({+ delivery: {+ deliveryMetadata: [metadata],+ kind: "deliver",+ payloads: [{ inputResponses: [{ optionId: "approve", requestId: "request-1" }] }],+ },+ parentWritable: createTestWritable(),+ sessionState: createStubSessionState({+ hasProxyInputRequests: true,+ sessionId: "parent-session",+ }),+ });++ expect(resumeHookMock).toHaveBeenCalledWith(+ "child-token",+ expect.objectContaining({ deliveryMetadata: [metadata] }),+ );+ });});describe("dispatchTurnStep", () => {diff --git a/packages/eve/src/execution/workflow-steps.ts b/packages/eve/src/execution/workflow-steps.tsindex 077068b9a..377c35cef 100644--- a/packages/eve/src/execution/workflow-steps.ts+++ b/packages/eve/src/execution/workflow-steps.ts@@ -1,6 +1,7 @@import { buildAdapterContext } from "#channel/adapter-context.js";import { callAdapterEventHandler, defaultDeliverResult } from "#channel/adapter.js";-import type { DeliverPayload, SessionAuthContext } from "#channel/types.js";+import type { DeliverHookPayload } from "#channel/types.js";+import { contextStorage } from "#context/container.js";import { dispatchStreamEventHooks } from "#context/hook-lifecycle.js";import { dispatchDynamicInstructionEvent } from "#context/dynamic-instruction-lifecycle.js";import { dispatchDynamicModelEvent } from "#context/dynamic-model-lifecycle.js";@@ -29,6 +30,11 @@ import {isHarnessBetweenTurns,setHarnessEmissionState,} from "#harness/emission.js";+import {+ channelDeliveryErrorCode,+ instrumentChannelDelivery,+} from "#harness/channel-delivery-instrumentation.js";+import { getInstrumentationRuntime } from "#harness/instrumentation-runtime.js";import { preserveSerializedInstrumentationState } from "#harness/instrumentation-state.js";import { RuntimeActionSettlementTimesKey } from "#harness/runtime-action-settlement-state.js";import { preserveSerializedAgentTraceState } from "#tracing/agent-trace-context-store.js";@@ -36,8 +42,8 @@ import { matchAuthorizationCallbacks } from "#execution/authorization-callback-mimport { readTurnSleepDurationMs } from "#harness/turn-sleep.js";import { isTurnCancellation, throwIfTurnAborted } from "#harness/turn-cancellation.js";import { setChannelContext } from "#execution/channel-context.js";-import { sendCommandToDelivery } from "#execution/session-command-wire.js";import { hasPendingInputBatch } from "#harness/input-requests.js";+import { activeTurnId } from "#harness/active-turn-id.js";import { coalesceTurnInputs } from "#harness/messages.js";import {getRuntimeActionKeysFromWorkflowInterrupt,@@ -70,29 +76,15 @@ import {createDurableSessionState,type DurableSessionState,readDurableSession,- replaceDurableSessionSnapshot,} from "#execution/durable-session-store.js";-import {- createTurnWorkflowInput,- type TurnStepInput,- type TurnWorkflowDispatchInput,-} from "#execution/durable-session-migrations/turn-workflow.js";+import type { TurnStepInput } from "#execution/durable-session-migrations/turn-workflow.js";import { buildRuntimeIdentity, createExecutionNodeStep } from "#execution/node-step.js";-import { routeDeliverPayload } from "#execution/subagent-hitl-proxy.js";-import { retireProxyInputRequests } from "#harness/proxy-input-requests.js";import { resolveEffectiveAgentRuntime } from "#execution/effective-agent-config.js";import { recordSubagentUsageSpans } from "#execution/subagent-usage-span.js";import { reconcileSessionContinuationToken } from "#execution/reconcile-session-continuation-token.js";import { hydrateDurableSession, refreshSessionFromTurnAgent } from "#execution/session.js";-import { buildTurnAttributes, readRootSessionId } from "#execution/eve-workflow-attributes.js";-import { normalizeEveAttributes } from "#runtime/attributes/normalize.js";import { resolveRuntimeCompiledArtifactsVersionedCacheKey } from "#runtime/cache-key.js";-import {- createWorkflowRuntime,- startWorkflowPreferLatest,- turnWorkflowReference,-} from "#execution/workflow-runtime.js";-import { resumeHook } from "#internal/workflow/runtime.js";+import { createWorkflowRuntime } from "#execution/workflow-runtime.js";const TASK_DONE_WITH_PENDING_INPUT_ERROR_MESSAGE ="Task mode cannot complete while input requests remain pending.";@@ -226,7 +218,37 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResudurable: durableSession,turnAgent: effectiveAgent.turnAgent,});+ const instrumentation = getInstrumentationRuntime();+ const initialEmissionState = getHarnessEmissionState(initialSession.state);++ if (rawInput.input?.kind === "deliver") {+ await contextStorage.run(ctx, () =>+ instrumentChannelDelivery({+ agentName: bundle.turnAgent.id,+ ctx,+ delivery: rawInput.input as DeliverHookPayload,+ hooks: instrumentation?.hooks,+ rootSessionId: initialSession.rootSessionId ?? initialSession.sessionId,+ sequence: initialEmissionState.sequence,+ sessionId: initialSession.sessionId,+ turnId: activeTurnId(initialEmissionState),+ }),+ );+ }+ const failChannelDeliveries = async (error: unknown): Promise<void> => {+ await contextStorage.run(ctx, () =>+ instrumentChannelDelivery({+ ctx,+ error,+ errorCode: channelDeliveryErrorCode(error),+ hooks: instrumentation?.hooks,+ includeTurn: false,+ outcome: "failed",+ }),+ );+ await instrumentation?.forceFlush();+ };const adapterCtx = buildAdapterContext(adapter, ctx);// Run the adapter's deliver hook for each queued payload and@@ -234,14 +256,19 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResulet resolved: StepInput | undefined;if (input.input?.kind === "deliver") {const results: StepInput[] = [];- for (const payload of input.input.payloads) {- const result = adapter.deliver- ? await adapter.deliver(payload, adapterCtx)- : defaultDeliverResult(payload);-- if (result !== undefined && result !== null) {- results.push(result);+ try {+ for (const payload of input.input.payloads) {+ const result = adapter.deliver+ ? await adapter.deliver(payload, adapterCtx)+ : defaultDeliverResult(payload);++ if (result !== undefined && result !== null) {+ results.push(result);+ }}+ } catch (error) {+ await failChannelDeliveries(error);+ throw error;}resolved = results.length === 0 ? undefined : results.reduce(coalesceTurnInputs);} else if (input.input?.kind === "runtime-action-result") {@@ -263,6 +290,15 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResu// that only edits a message). Re-park without a model turn; skip// the snapshot write when the session itself is unchanged.if (input.input?.kind === "deliver" && resolved === undefined) {+ await contextStorage.run(ctx, () =>+ instrumentChannelDelivery({+ ctx,+ hooks: instrumentation?.hooks,+ includeTurn: false,+ outcome: "completed",+ }),+ );+ await instrumentation?.forceFlush();const rekeyed = reconcileSessionContinuationToken(ctx, initialSession);const nextSerializedContext = serializeContext(ctx);const nextState =@@ -290,34 +326,39 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResuturnAgent: effectiveAgent.turnAgent,};const runtimeIdentity = buildRuntimeIdentity(effectiveNode);- const deploymentId = process.env.VERCEL_DEPLOYMENT_ID?.trim();- const dynamicRuntimeRevision = deploymentId- ? `deployment:${deploymentId}`- : await resolveRuntimeCompiledArtifactsVersionedCacheKey(bundle.compiledArtifactsSource);- const sessionStarted = getHarnessEmissionState(initialSession.state).sessionStarted;-- if (!sessionStarted) {- ctx.set(SessionDynamicSubagentRuntimeRevisionKey, dynamicRuntimeRevision);- ctx.set(SessionDynamicToolRuntimeRevisionKey, dynamicRuntimeRevision);- } else {- const refreshEvent = createSessionStartedEvent({ runtime: runtimeIdentity });- await Promise.all([- refreshDynamicSessionSubagentsForRuntimeRevision({- ctx,- resolvers: dynamicSubagentResolvers,- event: refreshEvent,- messages: initialSession.history,- persistentSessions: persistentSubagentSessions,- runtimeRevision: dynamicRuntimeRevision,- }),- refreshDynamicSessionToolsForRuntimeRevision({- ctx,- resolvers: dynamicToolResolvers,- event: refreshEvent,- messages: initialSession.history,- runtimeRevision: dynamicRuntimeRevision,- }),- ]);+ try {+ const deploymentId = process.env.VERCEL_DEPLOYMENT_ID?.trim();+ const dynamicRuntimeRevision = deploymentId+ ? `deployment:${deploymentId}`+ : await resolveRuntimeCompiledArtifactsVersionedCacheKey(bundle.compiledArtifactsSource);+ const sessionStarted = initialEmissionState.sessionStarted;++ if (!sessionStarted) {+ ctx.set(SessionDynamicSubagentRuntimeRevisionKey, dynamicRuntimeRevision);+ ctx.set(SessionDynamicToolRuntimeRevisionKey, dynamicRuntimeRevision);+ } else {+ const refreshEvent = createSessionStartedEvent({ runtime: runtimeIdentity });+ await Promise.all([+ refreshDynamicSessionSubagentsForRuntimeRevision({+ ctx,+ resolvers: dynamicSubagentResolvers,+ event: refreshEvent,+ messages: initialSession.history,+ persistentSessions: persistentSubagentSessions,+ runtimeRevision: dynamicRuntimeRevision,+ }),+ refreshDynamicSessionToolsForRuntimeRevision({+ ctx,+ resolvers: dynamicToolResolvers,+ event: refreshEvent,+ messages: initialSession.history,+ runtimeRevision: dynamicRuntimeRevision,+ }),+ ]);+ }+ } catch (error) {+ await failChannelDeliveries(error);+ throw error;}const writer = input.parentWritable.getWriter();@@ -450,7 +491,10 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResureturn runHarnessStep(schemaSession, resolved);});} catch (error) {- if (!isTurnCancellation(error)) throw error;+ if (!isTurnCancellation(error)) {+ await failChannelDeliveries(error);+ throw error;+ }writer.releaseLock();// Both carve-outs exist because the cancellation epilogue still publishes// `turn.cancelled` downstream, and the open spans and provider state it@@ -618,79 +662,3 @@ export function resolveEffectiveOutputSchema(input: {return session;}--export type RoutedDeliverResult =- | {- readonly kind: "cancel-turn";- readonly sessionState: DurableSessionState;- }- | {- readonly kind: "continue";- /** `undefined` when the entire payload was routed to descendants. */- readonly remainder: DeliverPayload | undefined;- readonly sessionState: DurableSessionState;- };--/**- * Splits an inbound deliver payload into parent-local and- * proxied-child buckets and forwards the child buckets via- * `resumeHook`. Successfully forwarded request IDs are retired in the- * returned snapshot so later deliveries cannot route through stale entries.- */-export async function routeProxiedDeliverStep(input: {- readonly auth?: SessionAuthContext | null;- readonly parentWritable: WritableStream<Uint8Array>;- readonly payload: DeliverPayload;- readonly sessionState: DurableSessionState;-}): Promise<RoutedDeliverResult> {- "use step";-- let durableSession = await readDurableSession(input.sessionState);- const routed = routeDeliverPayload({- payload: input.payload,- state: durableSession.state,- });-- for (const forChild of routed.forChildren) {- // Children are pinned to their own deployments, so proxied deliveries- // must cross this hook in the same durable envelope as channel sends.- await resumeHook(- forChild.childContinuationToken,- sendCommandToDelivery({ auth: input.auth, kind: "send", payload: forChild.payload }),- );- durableSession = retireProxyInputRequests(durableSession, forChild.retireRequestIds);- }-- const sessionState =- routed.forChildren.length === 0- ? input.sessionState- : replaceDurableSessionSnapshot({ session: durableSession, state: input.sessionState });-- return routed.parentAction === undefined- ? { kind: "continue", remainder: routed.forSelf, sessionState }- : { ...routed.parentAction, sessionState };-}--/** Starts a per-turn child workflow for the current driver session. */-export async function dispatchTurnStep(- input: TurnWorkflowDispatchInput,-): Promise<{ readonly runId: string }> {- "use step";-- const run = await startWorkflowPreferLatest(- turnWorkflowReference,- [createTurnWorkflowInput(input)],- {- allowReservedAttributes: true,- attributes: normalizeEveAttributes(- buildTurnAttributes({- parentSessionId: input.sessionState.sessionId,- requestId: input.delivery.kind === "deliver" ? input.delivery.requestId : undefined,- rootSessionId: readRootSessionId(input.serializedContext) ?? input.sessionState.sessionId,- }),- ),- },- );-- return { runId: run.runId };-}diff --git a/packages/eve/src/harness/channel-delivery-instrumentation.test.ts b/packages/eve/src/harness/channel-delivery-instrumentation.test.tsnew file mode 100644index 000000000..e9ab1ee41--- /dev/null+++ b/packages/eve/src/harness/channel-delivery-instrumentation.test.ts@@ -0,0 +1,78 @@+import { describe, expect, it } from "vitest";++import { ContextContainer, contextStorage } from "#context/container.js";+import { ParentTraceContextKey } from "#context/keys.js";+import { instrumentChannelDelivery } from "#harness/channel-delivery-instrumentation.js";+import {+ createInstrumentationHooks,+ type InstrumentationEvent,+} from "#harness/instrumentation-lifecycle.js";++describe("channel delivery instrumentation", () => {+ it("projects only known content and redacts it from metadata providers", async () => {+ const content: InstrumentationEvent[] = [];+ const metadata: InstrumentationEvent[] = [];+ const hooks = createInstrumentationHooks([+ {+ capture: "content",+ events: {+ "channel.delivery.started": (event) => {+ content.push(event);+ },+ },+ name: "content",+ },+ {+ events: {+ "channel.delivery.started": (event) => {+ metadata.push(event);+ },+ },+ name: "metadata",+ },+ ]);+ const ctx = new ContextContainer();+ const parentTraceContext = {+ isRemote: true,+ spanId: "2222222222222222",+ traceFlags: 1,+ traceId: "11111111111111111111111111111111",+ };+ ctx.set(ParentTraceContextKey, parentTraceContext);++ await contextStorage.run(ctx, async () => {+ await instrumentChannelDelivery({+ ctx,+ delivery: {+ deliveryMetadata: [+ {+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-1",+ payloadIndex: 0,+ },+ ],+ kind: "deliver",+ payloads: [{ interaction: { secret: true }, message: "hello" }],+ },+ hooks,+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn_0",+ });+ await instrumentChannelDelivery({+ ctx,+ hooks,+ includeTurn: true,+ outcome: "completed",+ });+ });++ expect(content[0]).toMatchObject({ input: { message: "hello" }, parentTraceContext });+ expect((content[0] as { input?: Record<string, unknown> }).input).not.toHaveProperty(+ "interaction",+ );+ expect(metadata[0]).toMatchObject({ input: undefined });+ });+});diff --git a/packages/eve/src/harness/channel-delivery-instrumentation.ts b/packages/eve/src/harness/channel-delivery-instrumentation.tsnew file mode 100644index 000000000..2f68322e3--- /dev/null+++ b/packages/eve/src/harness/channel-delivery-instrumentation.ts@@ -0,0 +1,134 @@+import type { DeliverHookPayload, DeliverPayload } from "#channel/types.js";+import type { AlsContext } from "#context/container.js";+import {+ ActiveChannelDeliveriesKey,+ ParentTraceContextKey,+ type ActiveChannelDelivery,+} from "#context/keys.js";+import type {+ InstrumentationChannelDeliveryOutcome,+ InstrumentationChannelDeliveryTerminalEvent,+ InstrumentationHooks,+} from "#harness/instrumentation-lifecycle.js";+import { channelDeliveryIdempotencyKey } from "#harness/instrumentation-lifecycle.js";++interface ChannelDeliveryStartInstrumentation {+ readonly agentName?: string;+ readonly ctx: AlsContext;+ readonly delivery: DeliverHookPayload;+ readonly hooks: InstrumentationHooks | undefined;+ readonly rootSessionId: string;+ readonly sequence: number;+ readonly sessionId: string;+ readonly turnId: string;+}++interface ChannelDeliveryTerminalInstrumentation {+ readonly ctx: AlsContext;+ readonly error?: unknown;+ readonly errorCode?: string;+ readonly hooks: InstrumentationHooks | undefined;+ readonly includeTurn: boolean;+ readonly outcome: InstrumentationChannelDeliveryOutcome;+}++export function instrumentChannelDelivery(+ input: ChannelDeliveryStartInstrumentation,+): Promise<void>;+export function instrumentChannelDelivery(+ input: ChannelDeliveryTerminalInstrumentation,+): Promise<void>;+export async function instrumentChannelDelivery(+ input: ChannelDeliveryStartInstrumentation | ChannelDeliveryTerminalInstrumentation,+): Promise<void> {+ if (!("delivery" in input)) {+ const active = input.ctx.get(ActiveChannelDeliveriesKey);+ if (active === undefined || input.hooks === undefined) return;+ const type =+ `channel.delivery.${input.outcome}` as InstrumentationChannelDeliveryTerminalEvent["type"];+ for (const item of active) {+ await input.hooks.publish({+ agentName: item.agentName,+ delivery: item.delivery,+ error: input.error,+ errorCode: input.errorCode,+ idempotencyKey: channelDeliveryIdempotencyKey(item.sessionId, item.delivery.deliveryId),+ outcome: input.outcome,+ rootSessionId: item.rootSessionId,+ sequence: input.includeTurn ? item.sequence : undefined,+ sessionId: item.sessionId,+ turnId: input.includeTurn ? item.turnId : undefined,+ type,+ });+ }+ input.ctx.delete(ActiveChannelDeliveriesKey);+ return;+ }++ if (input.hooks === undefined || input.delivery.deliveryMetadata === undefined) return;++ const active: ActiveChannelDelivery[] = [];+ for (const metadata of input.delivery.deliveryMetadata) {+ const payload = input.delivery.payloads[metadata.payloadIndex];+ const delivery = {+ channelKind: metadata.channelKind,+ channelName: metadata.channelName,+ deliveryId: metadata.deliveryId,+ requestId: metadata.requestId,+ requestTraceContext: metadata.requestTraceContext,+ };+ const deliveryInput =+ !input.hooks.capturesContent || payload === undefined+ ? undefined+ : projectDeliveryInput(payload);+ const item: ActiveChannelDelivery = {+ agentName: input.agentName,+ delivery,+ rootSessionId: input.rootSessionId,+ sequence: input.sequence,+ sessionId: input.sessionId,+ turnId: input.turnId,+ };+ active.push(item);+ await input.hooks.publish({+ agentName: item.agentName,+ delivery,+ idempotencyKey: channelDeliveryIdempotencyKey(input.sessionId, delivery.deliveryId),+ input: deliveryInput,+ parentTraceContext: input.ctx.get(ParentTraceContextKey),+ rootSessionId: input.rootSessionId,+ sessionId: input.sessionId,+ type: "channel.delivery.started",+ });+ }+ if (active.length > 0) input.ctx.set(ActiveChannelDeliveriesKey, active);+}++function projectDeliveryInput(payload: DeliverPayload) {+ const projected: {+ context?: DeliverPayload["context"];+ inputResponses?: Array<{ optionId?: string; text?: string }>;+ message?: DeliverPayload["message"];+ outputSchema?: DeliverPayload["outputSchema"];+ } = {};+ if (payload.context !== undefined) projected.context = payload.context;+ if (payload.message !== undefined) projected.message = payload.message;+ if (payload.outputSchema !== undefined) projected.outputSchema = payload.outputSchema;+ if (payload.inputResponses !== undefined) {+ projected.inputResponses = payload.inputResponses.map((response) => {+ const item: { optionId?: string; text?: string } = {};+ if (response.optionId !== undefined) item.optionId = response.optionId;+ if (response.text !== undefined) item.text = response.text;+ return item;+ });+ }+ return projected;+}++export function channelDeliveryErrorCode(error: unknown): string {+ if (typeof error === "object" && error !== null && "code" in error) {+ const code = (error as { readonly code?: unknown }).code;+ if (typeof code === "string" && code.length > 0) return code;+ }+ return "CHANNEL_DELIVERY_FAILED";+}diff --git a/packages/eve/src/harness/instrumentation-content.ts b/packages/eve/src/harness/instrumentation-content.tsindex 264311808..516c23c90 100644--- a/packages/eve/src/harness/instrumentation-content.ts+++ b/packages/eve/src/harness/instrumentation-content.ts@@ -3,6 +3,8 @@ import type { InstrumentationEvent } from "#harness/instrumentation-lifecycle.js/** Returns an immutable event projection with conversation content removed. */export function withoutInstrumentationContent(event: InstrumentationEvent): InstrumentationEvent {switch (event.type) {+ case "channel.delivery.started":+ return Object.freeze({ ...event, input: undefined });case "action.started":return Object.freeze({ ...event, input: undefined });case "action.completed":@@ -30,6 +32,7 @@ export function withoutInstrumentationContent(event: InstrumentationEvent): Instcase "step.attempt.failed":case "tool.call.failed":case "turn.failed":+ case "channel.delivery.failed":return Object.freeze({ ...event, error: undefined });default:return event;diff --git a/packages/eve/src/harness/instrumentation-dispatch.ts b/packages/eve/src/harness/instrumentation-dispatch.tsindex 8c0ba153f..19522a254 100644--- a/packages/eve/src/harness/instrumentation-dispatch.ts+++ b/packages/eve/src/harness/instrumentation-dispatch.ts@@ -233,6 +233,14 @@ async function withTimeout(/** Model and SDK tool children are scoped to an attempt; durable pairs are not. */function stateOwner(event: InstrumentationEvent): InstrumentationStateOwner {+ if (+ event.type === "channel.delivery.started" ||+ event.type === "channel.delivery.cancelled" ||+ event.type === "channel.delivery.completed" ||+ event.type === "channel.delivery.failed"+ ) {+ return { sessionId: event.sessionId, turnId: event.turnId };+ }if (!("scope" in event)) return {};if (event.type.startsWith("action.") || event.type.startsWith("input.")) {return { sessionId: event.scope.sessionId, turnId: event.scope.turnId };diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.tsindex b99f0a38b..6c9b892ef 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts@@ -154,6 +154,56 @@ export function inputIdempotencyKey(sessionId: string, turnId: string, requestIdreturn `input:${sessionId}:${turnId}:${requestId}`;}+export function channelDeliveryIdempotencyKey(sessionId: string, deliveryId: string): string {+ return `channel-delivery:${sessionId}:${deliveryId}`;+}++export interface InstrumentationChannelDeliveryRef {+ readonly channelKind: string;+ readonly channelName: string;+ readonly deliveryId: string;+ readonly requestId?: string;+ readonly requestTraceContext?: InstrumentationTraceContext;+}++/** Known framework delivery input; adapter-specific fields are never projected. */+export interface InstrumentationChannelDeliveryInput {+ readonly context?: readonly string[];+ readonly inputResponses?: readonly InstrumentationInputResponse[];+ readonly message?: unknown;+ readonly outputSchema?: unknown;+}++interface InstrumentationChannelDeliveryScope {+ readonly agentName?: string;+ readonly delivery: InstrumentationChannelDeliveryRef;+ readonly idempotencyKey: string;+ readonly rootSessionId: string;+ readonly sequence?: number;+ readonly sessionId: string;+ readonly turnId?: string;+}++export interface InstrumentationChannelDeliveryStartedEvent extends InstrumentationChannelDeliveryScope {+ readonly parentTraceContext?: InstrumentationTraceContext;+ readonly type: "channel.delivery.started";+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly input?: InstrumentationChannelDeliveryInput;+}++export type InstrumentationChannelDeliveryOutcome = "cancelled" | "completed" | "failed";++export interface InstrumentationChannelDeliveryTerminalEvent extends InstrumentationChannelDeliveryScope {+ readonly type:+ | "channel.delivery.cancelled"+ | "channel.delivery.completed"+ | "channel.delivery.failed";+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly error?: unknown;+ readonly errorCode?: string;+ readonly outcome: InstrumentationChannelDeliveryOutcome;+}+/** Framework-owned reason an agent suspended for user input. */export type InstrumentationInputKind = "question" | "session-limit" | "tool-approval";@@ -481,6 +531,10 @@ export interface InstrumentationProviderDefinition {/** Defaults to `"metadata"`. See {@link InstrumentationCapture}. */readonly capture?: InstrumentationCapture;readonly events?: {+ readonly "channel.delivery.started"?: InstrumentationEventHandler<InstrumentationChannelDeliveryStartedEvent>;+ readonly "channel.delivery.cancelled"?: InstrumentationEventHandler<InstrumentationChannelDeliveryTerminalEvent>;+ readonly "channel.delivery.completed"?: InstrumentationEventHandler<InstrumentationChannelDeliveryTerminalEvent>;+ readonly "channel.delivery.failed"?: InstrumentationEventHandler<InstrumentationChannelDeliveryTerminalEvent>;readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;readonly "step.attempt.failed"?: InstrumentationEventHandler<InstrumentationStepAttemptFailedEvent>;@@ -527,6 +581,8 @@ export type InstrumentationHooksInput =/** Events that pair a start with its terminal under one `idempotencyKey`. */export type InstrumentationCorrelatedEvent =+ | InstrumentationChannelDeliveryStartedEvent+ | InstrumentationChannelDeliveryTerminalEvent| InstrumentationInputRequestedEvent| InstrumentationInputResolvedEvent| InstrumentationActionStartedEventdiff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.tsindex 82869bcc5..bbe9b3a79 100644--- a/packages/eve/src/harness/instrumentation-native-events.test.ts+++ b/packages/eve/src/harness/instrumentation-native-events.test.ts@@ -1,6 +1,7 @@import { describe, expect, it } from "vitest";import { ContextContainer, contextStorage } from "#context/container.js";+import { ActiveChannelDeliveriesKey } from "#context/keys.js";import { deserializeContext, serializeContext } from "#context/serialize.js";import {createActionResultEvent,@@ -11,13 +12,18 @@ import {createStepCompletedEvent,createStepStartedEvent,createTurnCompletedEvent,+ createTurnCancelledEvent,+ createTurnFailedEvent,createTurnStartedEvent,} from "#protocol/message.js";import {createInstrumentationHandleEvent,publishInputResolutions,} from "#harness/instrumentation-native-events.js";-import type { InstrumentationHooks } from "#harness/instrumentation-lifecycle.js";+import type {+ InstrumentationEvent,+ InstrumentationHooks,+} from "#harness/instrumentation-lifecycle.js";import {actionIdempotencyKey,inputIdempotencyKey,@@ -27,6 +33,57 @@ import {import { RuntimeActionSettlementTimesKey } from "#harness/runtime-action-settlement-state.js";describe("createInstrumentationHandleEvent", () => {+ it.each([+ ["completed", createTurnCompletedEvent({ sequence: 0, turnId: "turn_0" })],+ ["cancelled", createTurnCancelledEvent({ sequence: 0, turnId: "turn_0" })],+ [+ "failed",+ createTurnFailedEvent({+ code: "MODEL_CALL_FAILED",+ message: "failed",+ sequence: 0,+ turnId: "turn_0",+ }),+ ],+ ] as const)("terminates active channel deliveries when a turn is %s", async (outcome, event) => {+ const ctx = new ContextContainer();+ ctx.set(ActiveChannelDeliveriesKey, [+ {+ delivery: {+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-1",+ },+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn_0",+ },+ ]);+ const events: InstrumentationEvent[] = [];+ const handleEvent = createInstrumentationHandleEvent({+ handleEvent: async () => {},+ hooks: {+ capturesContent: false,+ publish: async (published) => {+ events.push(published);+ },+ },+ sessionId: "session-1",+ })!;++ await contextStorage.run(ctx, () => handleEvent(event));++ expect(+ events.find((published) => published.type.startsWith("channel.delivery.")),+ ).toMatchObject({+ errorCode: outcome === "failed" ? "MODEL_CALL_FAILED" : undefined,+ outcome,+ type: `channel.delivery.${outcome}`,+ });+ expect(ctx.get(ActiveChannelDeliveriesKey)).toBeUndefined();+ });+it("publishes native lifecycle transitions after durable handling", async () => {const order: string[] = [];const hooks: InstrumentationHooks = {diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.tsindex 4b4db594b..de1e321ea 100644--- a/packages/eve/src/harness/instrumentation-native-events.ts+++ b/packages/eve/src/harness/instrumentation-native-events.ts@@ -1,5 +1,6 @@import type { UnstampedMessageStreamEvent } from "#protocol/message.js";import { contextStorage } from "#context/container.js";+import { instrumentChannelDelivery } from "#harness/channel-delivery-instrumentation.js";import type {InstrumentationActionFailedEvent,InstrumentationActionStartedEvent,@@ -31,6 +32,7 @@ import type { RuntimeActionRequest, RuntimeActionResult } from "#runtime/actionsexport interface CreateInstrumentationHandleEventInput {readonly agentName?: string;+ readonly channelKind?: string;readonly getAttemptScope?: () => InstrumentationAttemptScope | undefined;readonly handleEvent?: HandleEventFn;readonly hooks?: InstrumentationHooks;@@ -57,6 +59,39 @@ export function createInstrumentationHandleEvent(await handleEvent(event, messages);const lifecycleEvent = toLifecycleEvent(event, input, activeTurnId);if (event.type === "turn.started") activeTurnId = event.data.turnId;+ if (+ event.type === "turn.cancelled" ||+ event.type === "turn.completed" ||+ event.type === "turn.failed" ||+ event.type === "session.completed" ||+ event.type === "session.failed"+ ) {+ const ctx = contextStorage.getStore();+ if (ctx !== undefined) {+ await instrumentChannelDelivery({+ ctx,+ error:+ event.type === "turn.failed" || event.type === "session.failed"+ ? new Error(event.data.message)+ : undefined,+ errorCode:+ event.type === "turn.failed" || event.type === "session.failed"+ ? event.data.code+ : undefined,+ hooks,+ includeTurn:+ event.type === "turn.cancelled" ||+ event.type === "turn.completed" ||+ event.type === "turn.failed",+ outcome:+ event.type === "turn.failed" || event.type === "session.failed"+ ? "failed"+ : event.type === "turn.cancelled"+ ? "cancelled"+ : "completed",+ });+ }+ }if (lifecycleEvent !== undefined) await hooks.publish(lifecycleEvent);if (event.type === "actions.requested") {await publishActionStarts(event, input, hooks, publishedActions);@@ -257,6 +292,7 @@ function toLifecycleEvent(case "session.started":return {agentName: input.agentName,+ channelKind: input.channelKind,idempotencyKey: sessionIdempotencyKey(input.sessionId),parentTraceContext: input.parentTraceContext,rootSessionId: input.rootSessionId ?? input.sessionId,diff --git a/packages/eve/src/harness/instrumentation-state.ts b/packages/eve/src/harness/instrumentation-state.tsindex 3ea17a7be..319b48bfa 100644--- a/packages/eve/src/harness/instrumentation-state.ts+++ b/packages/eve/src/harness/instrumentation-state.ts@@ -1,4 +1,5 @@import { contextStorage, loadContext } from "#context/container.js";+import { ActiveChannelDeliveriesKey } from "#context/keys.js";import { ContextKey } from "#context/key.js";import { type JsonValue, parseJsonValue } from "#shared/json.js";import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";@@ -81,6 +82,7 @@ export function preserveSerializedInstrumentationState(InstrumentationStateKey,InstrumentationActionScopeKey,InstrumentationInputScopeKey,+ ActiveChannelDeliveriesKey,]) {const state = interrupted[key.name];if (state !== undefined) preserved = { ...preserved, [key.name]: state };diff --git a/packages/eve/src/harness/messages.test.ts b/packages/eve/src/harness/messages.test.tsindex daf28ba26..b50d7e4eb 100644--- a/packages/eve/src/harness/messages.test.ts+++ b/packages/eve/src/harness/messages.test.ts@@ -52,6 +52,33 @@ describe("coalesceDeliveries", () => {]),).toThrow("Cannot coalesce deliveries from different turns.");});++ it("reindexes all delivery metadata while coalescing", () => {+ const metadata = (deliveryId: string) => ({+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId,+ payloadIndex: 0,+ });++ const result = coalesceDeliveries([+ {+ deliveryMetadata: [metadata("delivery-1")],+ kind: "deliver" as const,+ payloads: [{ message: "first" }],+ },+ {+ deliveryMetadata: [metadata("delivery-2")],+ kind: "deliver" as const,+ payloads: [{ message: "second" }],+ },+ ]);++ expect(result.deliveryMetadata).toEqual([+ metadata("delivery-1"),+ { ...metadata("delivery-2"), payloadIndex: 1 },+ ]);+ });});describe("coalesceTurnInputs", () => {diff --git a/packages/eve/src/harness/messages.ts b/packages/eve/src/harness/messages.tsindex 2479fdb3a..ae093f21f 100644--- a/packages/eve/src/harness/messages.ts+++ b/packages/eve/src/harness/messages.ts@@ -1,6 +1,11 @@import type { ModelMessage, TextPart, UserContent } from "ai";-import type { DeliverPayload, SessionAuthContext, TurnCaller } from "#channel/types.js";+import type {+ ChannelDeliveryMetadataEntry,+ DeliverPayload,+ SessionAuthContext,+ TurnCaller,+} from "#channel/types.js";import type { InputResponse } from "#runtime/input/types.js";import type { StepInput } from "#harness/types.js";@@ -216,6 +221,7 @@ function toUserContentArray(value: string | UserContent): UserContentArray {interface DeliverLike {readonly auth?: SessionAuthContext | null;readonly caller?: TurnCaller;+ readonly deliveryMetadata?: readonly ChannelDeliveryMetadataEntry[];readonly kind: "deliver";readonly payloads: readonly DeliverPayload[];}@@ -240,8 +246,10 @@ export function coalesceDeliveries<T extends DeliverLike>(items: readonly T[]):let auth = first.auth;let caller = first.caller;const payloads = [...first.payloads];+ const deliveryMetadata = [...(first.deliveryMetadata ?? [])];for (const item of rest) {+ const payloadOffset = payloads.length;if (item.auth !== undefined) {auth = item.auth;}@@ -252,7 +260,19 @@ export function coalesceDeliveries<T extends DeliverLike>(items: readonly T[]):caller = item.caller;}payloads.push(...item.payloads);+ deliveryMetadata.push(+ ...(item.deliveryMetadata ?? []).map((entry) => ({+ ...entry,+ payloadIndex: entry.payloadIndex + payloadOffset,+ })),+ );}- return { ...first, auth, caller, payloads };+ return {+ ...first,+ auth,+ caller,+ deliveryMetadata: deliveryMetadata.length === 0 ? undefined : deliveryMetadata,+ payloads,+ };}diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex 5e99ea925..e2c5c6203 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -32,6 +32,7 @@ import { formatLanguageModelGatewayId } from "#internal/runtime-model.js";import { contextStorage } from "#context/container.js";import {AuthKey,+ ChannelInstrumentationKey,ParentSessionKey,ParentTraceContextKey,SessionCallbackKey,@@ -619,6 +620,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {let activeAttemptScope: InstrumentationAttemptScope | undefined;const emit = createInstrumentationHandleEvent({agentName: config.runtimeIdentity?.agentName,+ channelKind: store?.get(ChannelInstrumentationKey)?.kind,getAttemptScope: () => activeAttemptScope,handleEvent: baseEmit,hooks: config.instrumentation?.hooks,diff --git a/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts b/packages/eve/src/internal/nitro/routes/channel-dispatch.test.tsindex 76f0a3769..090c86f4f 100644--- a/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts+++ b/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts@@ -814,6 +814,54 @@ describe("dispatchChannelRequest tracing", () => {expect(parentSpanId(nested!)).toBe(server!.spanContext().spanId);});+ it("captures the created SERVER span context on channel delivery metadata", async () => {+ const tracedRuntime = createRuntime({+ dispatchContinuation: vi+ .fn()+ .mockResolvedValue({ sessionId: "session-1", status: "accepted" }),+ });+ mockedResolveNitroChannelRuntimeBundle.mockResolvedValue({+ channels: [+ slackChannel(async () => new Response("unused"), {+ adapter: { kind: "channel:slack" },+ handler: async (_request, args) => {+ await args.from("thread-1").send("hello", { auth: null });+ return new Response("ok");+ },+ }),+ ],+ runtime: tracedRuntime,+ });++ await dispatchChannelRequest(+ createEvent({+ headers: {+ traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",+ "x-vercel-id": "iad1::request-1",+ },+ method: "POST",+ waitUntil: vi.fn(),+ }),+ "POST /slack",+ {} as never,+ );++ const [server] = await finishedSpans();+ const command = vi.mocked(tracedRuntime.dispatchContinuation).mock.calls[0]?.[0].command;+ const delivery = command?.kind === "send" ? command.delivery : undefined;+ expect(delivery).toMatchObject({+ channelKind: "channel:slack",+ channelName: "slack",+ requestId: "iad1::request-1",+ requestTraceContext: {+ spanId: server!.spanContext().spanId,+ traceFlags: server!.spanContext().traceFlags,+ traceId: server!.spanContext().traceId,+ },+ });+ expect(delivery?.requestTraceContext?.spanId).not.toBe("b7ad6b7169203331");+ });+it("adopts an incoming traceparent as the request span's parent", async () => {const traceId = "0af7651916cd43dd8448eb211c80319c";const parentId = "b7ad6b7169203331";diff --git a/packages/eve/src/internal/nitro/routes/channel-dispatch.ts b/packages/eve/src/internal/nitro/routes/channel-dispatch.tsindex 616bf80af..3708d1914 100644--- a/packages/eve/src/internal/nitro/routes/channel-dispatch.ts+++ b/packages/eve/src/internal/nitro/routes/channel-dispatch.ts@@ -1,9 +1,11 @@import type { H3Event } from "nitro";+import type { Span } from "#compiled/@opentelemetry/api/index.js";import type { RouteContext } from "#public/definitions/channel.js";import { getChannelInstrumentationKind } from "#channel/compiled-channel.js";import { createCrossChannelToFn, toCrossChannelTargets } from "#channel/cross-channel-receive.js";import type { RouteHandlerArgs, WebSocketRouteHooks } from "#channel/routes.js";import { createChannelOperations } from "#channel/channel-operations.js";+import { createChannelDeliveryMetadata } from "#channel/delivery-metadata.js";import { createAttachSessionFn } from "#channel/session.js";import { createLogger, logError } from "#internal/logging.js";import { readTrustedDevelopmentClientAddress } from "#internal/nitro/dev-client-address.js";@@ -71,7 +73,14 @@ export async function dispatchChannelRequest(span?.setAttribute("eve.channel.kind", channelKind);}- const routeArgs = buildRouteArgs(event, bundle, matchedChannel.name, config);+ const routeArgs = buildRouteArgs(+ event,+ bundle,+ matchedChannel.name,+ channelKind ?? "channel",+ config,+ span,+ );let response: Response;@@ -132,7 +141,18 @@ export async function dispatchChannelWebSocketRequest(}const websocket = matchedChannel.websocket;- const routeArgs = buildRouteArgs(event, bundle, matchedChannel.name, config);+ const channelKind =+ getChannelInstrumentationKind(matchedChannel.definition) ??+ matchedChannel.adapter?.kind ??+ "channel";+ const routeArgs = buildRouteArgs(+ event,+ bundle,+ matchedChannel.name,+ channelKind,+ config,+ undefined,+ );try {const hooks = await withDevelopmentVercelOidcContext(@@ -182,7 +202,9 @@ function buildRouteArgs(event: H3Event,bundle: Awaited<ReturnType<typeof resolveNitroChannelRuntimeBundle>>,channelName: string,+ channelKind: string,config: NitroArtifactsConfig,+ requestSpan: Span | undefined,): BuiltRouteArgs {const requestId = readVercelRequestId(event.req.headers);const requestIp = extractRequestIp(event, config);@@ -198,15 +220,29 @@ function buildRouteArgs(};const channel = bundle.channels.find((candidate) => candidate.name === channelName);const adapter = channel?.adapter ?? { kind: "channel" };+ const requestSpanContext = requestSpan?.spanContext();+ const deliverySource = {+ channelKind,+ channelName,+ requestId,+ requestTraceContext:+ requestSpanContext === undefined+ ? undefined+ : {+ spanId: requestSpanContext.spanId,+ traceFlags: requestSpanContext.traceFlags,+ traceId: requestSpanContext.traceId,+ },+ };const channelOperations = createChannelOperations({adapter,channelName,- metadata: { requestId },+ metadata: deliverySource,runtime: bundle.runtime,turnPolicy: channel?.turnPolicy,});const attachSession = createAttachSessionFn(bundle.runtime, {- requestId,+ ...deliverySource,turnPolicy: channel?.turnPolicy,});const to = createCrossChannelToFn(bundle.runtime, toCrossChannelTargets(bundle.channels));@@ -231,6 +267,7 @@ function buildRouteArgs(...input,adapter,channelName,+ delivery: createChannelDeliveryMetadata(deliverySource),requestId,}),);diff --git a/packages/eve/src/public/instrumentation/provider.test.ts b/packages/eve/src/public/instrumentation/provider.test.tsindex 0fe3e5108..7752e0614 100644--- a/packages/eve/src/public/instrumentation/provider.test.ts+++ b/packages/eve/src/public/instrumentation/provider.test.ts@@ -60,6 +60,18 @@ describe("defineInstrumentation", () => {expect(isInstrumentationProvider(provider)).toBe(true);});+ it("exposes channel delivery lifecycle events", () => {+ const provider = defineInstrumentation({+ events: {+ "channel.delivery.started": (event) => void event.delivery.deliveryId,+ "channel.delivery.completed": (event) => void event.outcome,+ "channel.delivery.failed": (event) => void event.errorCode,+ },+ });++ expect(isInstrumentationProvider(provider)).toBe(true);+ });+it("preserves the authored fields", () => {const setup = (): void => {};const declaration = defineInstrumentation({ setup });diff --git a/packages/eve/src/public/instrumentation/provider.ts b/packages/eve/src/public/instrumentation/provider.tsindex 89a92ad1e..d3ac25b2f 100644--- a/packages/eve/src/public/instrumentation/provider.ts+++ b/packages/eve/src/public/instrumentation/provider.ts@@ -26,6 +26,11 @@ export type {InstrumentationActionStartedEvent,InstrumentationAttemptScope,InstrumentationCapture,+ InstrumentationChannelDeliveryInput,+ InstrumentationChannelDeliveryOutcome,+ InstrumentationChannelDeliveryRef,+ InstrumentationChannelDeliveryStartedEvent,+ InstrumentationChannelDeliveryTerminalEvent,InstrumentationContentPart,InstrumentationEvent,InstrumentationInputKind,diff --git a/packages/eve/src/tracing/agent-channel-delivery-instrumentation.ts b/packages/eve/src/tracing/agent-channel-delivery-instrumentation.tsnew file mode 100644index 000000000..8f90c2368--- /dev/null+++ b/packages/eve/src/tracing/agent-channel-delivery-instrumentation.ts@@ -0,0 +1,212 @@+import {+ ROOT_CONTEXT,+ SpanKind,+ SpanStatusCode,+ type Span,+ type SpanContext,+ type Tracer,+ trace,+} from "#compiled/@opentelemetry/api/index.js";++import type {+ InstrumentationChannelDeliveryStartedEvent,+ InstrumentationChannelDeliveryTerminalEvent,+ InstrumentationHandlerContext,+ InstrumentationProviderDefinition,+ InstrumentationSessionStartedEvent,+} from "#harness/instrumentation-lifecycle.js";+import { sessionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";+import type { JsonValue } from "#shared/json.js";+import { contentAttribute } from "#tracing/agent-otel-content.js";+import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import type { AgentSessionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";++interface ChannelDeliverySpanState {+ readonly inputAttribute?: string;+ readonly parent: SpanContext;+ readonly requestTraceContext?: SpanContext;+ readonly spanId: string;+ readonly startTimeMs: number;+ readonly window: number;+}++/** Builds durable channel delivery spans around the turn that consumes each request. */+export function createAgentChannelDeliveryInstrumentation(input: {+ readonly ensureSessionContext: (+ event: InstrumentationSessionStartedEvent,+ ) => Promise<AgentSessionTraceState>;+ readonly frameworkVersion: string;+ readonly idGenerator: AgentSpanIdGenerator;+ readonly recordInputs: boolean;+ readonly stateStore: AgentTraceStateStore;+ readonly tracer: Tracer;+}): Pick<+ NonNullable<InstrumentationProviderDefinition["events"]>,+ | "channel.delivery.cancelled"+ | "channel.delivery.completed"+ | "channel.delivery.failed"+ | "channel.delivery.started"+> {+ const onStarted = async (+ event: InstrumentationChannelDeliveryStartedEvent,+ ctx: InstrumentationHandlerContext,+ ): Promise<void> => {+ const session = await input.ensureSessionContext({+ agentName: event.agentName,+ channelKind: event.delivery.channelKind,+ idempotencyKey: sessionIdempotencyKey(event.sessionId),+ parentTraceContext: event.parentTraceContext,+ rootSessionId: event.rootSessionId,+ sessionId: event.sessionId,+ type: "session.started",+ });+ const inputAttribute = input.recordInputs ? contentAttribute(event.input, false) : undefined;+ const state: Record<string, JsonValue> = {+ parent: {+ isRemote: session.context.isRemote ?? false,+ spanId: session.context.spanId,+ traceFlags: session.context.traceFlags,+ traceId: session.context.traceId,+ },+ spanId: input.idGenerator.deriveSpanId(`channel-delivery:${event.idempotencyKey}`),+ startTimeMs: Date.now(),+ window: session.window,+ };+ if (inputAttribute !== undefined) state.inputAttribute = inputAttribute;+ if (event.delivery.requestTraceContext !== undefined) {+ state.requestTraceContext = {+ spanId: event.delivery.requestTraceContext.spanId,+ traceFlags: event.delivery.requestTraceContext.traceFlags,+ traceId: event.delivery.requestTraceContext.traceId,+ };+ }+ ctx.state.set(state);+ };++ const onTerminal = async (+ event: InstrumentationChannelDeliveryTerminalEvent,+ ctx: InstrumentationHandlerContext,+ ): Promise<void> => {+ const state = readState(ctx.state.get());+ if (state === undefined) return;+ const turn =+ event.turnId === undefined+ ? undefined+ : await input.stateStore.getTurn(event.sessionId, event.turnId);+ const session = await input.stateStore.getSession(event.sessionId);+ const parent =+ turn === undefined+ ? state.parent+ : {+ isRemote: turn.parentIsRemote ?? false,+ spanId: turn.parentSpanId,+ traceFlags: turn.context.traceFlags,+ traceId: turn.context.traceId,+ };+ const startTimeMs =+ turn === undefined ? state.startTimeMs : Math.max(state.startTimeMs, turn.startTimeMs);+ const requestLink = state.requestTraceContext;+ const span = input.idGenerator.withSpanId(state.spanId, () =>+ input.tracer.startSpan(+ "agent.channel.delivery",+ {+ attributes: {+ "agent.channel.delivery.id": event.delivery.deliveryId,+ "agent.channel.delivery.outcome": event.outcome,+ "agent.channel.kind": event.delivery.channelKind,+ "agent.channel.name": event.delivery.channelName,+ "agent.channel.request.id": event.delivery.requestId,+ "agent.framework.name": "eve",+ "agent.framework.version": input.frameworkVersion,+ "agent.name": event.agentName,+ "agent.root.session.id": event.rootSessionId,+ "agent.session.id": event.sessionId,+ "agent.session.window": session?.window ?? state.window,+ "agent.turn.id": event.turnId,+ "agent.turn.sequence": event.sequence,+ },+ kind: SpanKind.CONSUMER,+ links:+ requestLink === undefined+ ? undefined+ : [+ {+ attributes: { "eve.link.type": "channel.request" },+ context: requestLink,+ },+ ],+ startTime: startTimeMs,+ },+ contextFromSpanContext(parent),+ ),+ );+ span.addEvent("channel.delivery.started", undefined, startTimeMs);+ span.addEvent(event.type);+ if (state.inputAttribute !== undefined) {+ span.setAttribute("agent.channel.delivery.input", state.inputAttribute);+ }+ if (event.outcome === "failed") {+ const errorType =+ event.errorCode ?? (event.error instanceof Error ? event.error.name : "Error");+ span.setAttribute("error.type", errorType);+ if (event.errorCode !== undefined) {+ span.setAttribute("agent.channel.delivery.error.code", event.errorCode);+ }+ recordError(span, event.error);+ }+ span.end();+ };++ return {+ "channel.delivery.cancelled": onTerminal,+ "channel.delivery.completed": onTerminal,+ "channel.delivery.failed": onTerminal,+ "channel.delivery.started": onStarted,+ };+}++function readState(value: unknown): ChannelDeliverySpanState | undefined {+ if (!isRecord(value) || !isSpanContext(value.parent)) return undefined;+ if (+ typeof value.spanId !== "string" ||+ typeof value.startTimeMs !== "number" ||+ typeof value.window !== "number"+ ) {+ return undefined;+ }+ const requestTraceContext = isSpanContext(value.requestTraceContext)+ ? value.requestTraceContext+ : undefined;+ return {+ inputAttribute: typeof value.inputAttribute === "string" ? value.inputAttribute : undefined,+ parent: value.parent,+ requestTraceContext,+ spanId: value.spanId,+ startTimeMs: value.startTimeMs,+ window: value.window,+ };+}++function isSpanContext(value: unknown): value is SpanContext {+ return (+ isRecord(value) &&+ typeof value.spanId === "string" &&+ typeof value.traceFlags === "number" &&+ typeof value.traceId === "string"+ );+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function contextFromSpanContext(spanContext: SpanContext) {+ return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));+}++function recordError(span: Span, error: unknown): void {+ if (error instanceof Error) {+ span.recordException(error);+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });+ } else span.setStatus({ code: SpanStatusCode.ERROR });+}diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 2db1b4ad2..666649452 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -1,4 +1,4 @@-import { SpanStatusCode, trace as apiTrace } from "@opentelemetry/api";+import { SpanKind, SpanStatusCode, trace as apiTrace } from "@opentelemetry/api";import {BasicTracerProvider,InMemorySpanExporter,@@ -313,6 +313,261 @@ function nanos(hrTime: readonly [number, number]): bigint {}describe("createAgentOtelInstrumentation", () => {+ it.each([+ ["cancelled", SpanStatusCode.UNSET],+ ["failed", SpanStatusCode.ERROR],+ ] as const)("records a %s channel delivery with the expected status", async (outcome, status) => {+ const runtime = createRuntime();+ const ctx = new ContextContainer();+ const delivery = {+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: `delivery-${outcome}`,+ };+ const idempotencyKey = `channel-delivery:session-1:${delivery.deliveryId}`;++ await contextStorage.run(ctx, async () => {+ await runtime.hooks.publish({+ delivery,+ idempotencyKey,+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "channel.delivery.started",+ });+ await runtime.hooks.publish({+ delivery,+ error: outcome === "failed" ? new Error("failed") : undefined,+ errorCode: outcome === "failed" ? "CHANNEL_DELIVERY_FAILED" : undefined,+ idempotencyKey,+ outcome,+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: `channel.delivery.${outcome}`,+ });+ });++ const span = runtime.exporter+ .getFinishedSpans()+ .find((candidate) => candidate.name === "agent.channel.delivery")!;+ expect(span.status.code).toBe(status);+ expect(span.attributes["error.type"]).toBe(+ outcome === "failed" ? "CHANNEL_DELIVERY_FAILED" : undefined,+ );+ });++ it("maps channel delivery under the session window with an HTTP request link", async () => {+ const runtime = createRuntime();+ const ctx = new ContextContainer();+ const requestTraceContext = {+ spanId: "2222222222222222",+ traceFlags: 1,+ traceId: "22222222222222222222222222222222",+ };+ const started = {+ agentName: "support",+ delivery: {+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-1",+ requestId: "iad1::request-1",+ requestTraceContext,+ },+ idempotencyKey: "channel-delivery:session-1:delivery-1",+ input: { message: "private" },+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "channel.delivery.started" as const,+ };+ const completed = {+ agentName: started.agentName,+ delivery: started.delivery,+ idempotencyKey: started.idempotencyKey,+ outcome: "completed" as const,+ rootSessionId: started.rootSessionId,+ sequence: 0,+ sessionId: started.sessionId,+ turnId: "turn_0",+ type: "channel.delivery.completed" as const,+ };++ await contextStorage.run(ctx, async () => {+ await runtime.hooks.publish({+ agentName: "support",+ channelKind: "channel:slack",+ idempotencyKey: sessionIdempotencyKey("session-1"),+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "session.started",+ });+ await runtime.hooks.publish(started);+ await runtime.hooks.publish(completed);+ });++ const spans = runtime.exporter.getFinishedSpans();+ const session = spans.find((span) => span.name === "agent.session")!;+ const delivery = spans.find((span) => span.name === "agent.channel.delivery")!;+ expect(delivery.kind).toBe(SpanKind.CONSUMER);+ expect(delivery.parentSpanContext?.spanId).toBe(session.spanContext().spanId);+ expect(delivery.links).toEqual([+ expect.objectContaining({+ attributes: { "eve.link.type": "channel.request" },+ context: expect.objectContaining(requestTraceContext),+ }),+ ]);+ expect(delivery.attributes).toMatchObject({+ "agent.channel.delivery.id": "delivery-1",+ "agent.channel.delivery.input": JSON.stringify({ message: "private" }),+ "agent.channel.delivery.outcome": "completed",+ "agent.channel.kind": "channel:slack",+ "agent.channel.name": "slack",+ "agent.channel.request.id": "iad1::request-1",+ "agent.session.id": "session-1",+ "agent.turn.id": "turn_0",+ "agent.turn.sequence": 0,+ });++ await contextStorage.run(ctx, async () => {+ await runtime.hooks.publish(started);+ await runtime.hooks.publish(completed);+ });+ const replayed = runtime.exporter+ .getFinishedSpans()+ .filter((span) => span.name === "agent.channel.delivery");+ expect(replayed).toHaveLength(2);+ expect(replayed[1]!.spanContext().spanId).toBe(replayed[0]!.spanContext().spanId);+ });++ it("preserves a remote parent when delivery starts before the session event", async () => {+ const runtime = createRuntime();+ const ctx = new ContextContainer();+ const parentTraceContext = {+ isRemote: true as const,+ spanId: "2222222222222222",+ traceFlags: 1,+ traceId: "11111111111111111111111111111111",+ };+ const delivery = {+ channelKind: "http",+ channelName: "eve",+ deliveryId: "delivery-remote",+ };+ const idempotencyKey = `channel-delivery:remote-session:${delivery.deliveryId}`;++ await contextStorage.run(ctx, async () => {+ await runtime.hooks.publish({+ delivery,+ idempotencyKey,+ parentTraceContext,+ rootSessionId: "parent-session",+ sessionId: "remote-session",+ type: "channel.delivery.started",+ });+ await runtime.hooks.publish({+ agentName: "weather",+ channelKind: "http",+ idempotencyKey: sessionIdempotencyKey("remote-session"),+ parentTraceContext,+ rootSessionId: "parent-session",+ sessionId: "remote-session",+ type: "session.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: turnIdempotencyKey("remote-session", "turn-remote"),+ parentTraceContext,+ rootSessionId: "parent-session",+ sequence: 0,+ sessionId: "remote-session",+ turnId: "turn-remote",+ type: "turn.started",+ });+ await runtime.hooks.publish({+ delivery,+ idempotencyKey,+ outcome: "completed",+ rootSessionId: "parent-session",+ sequence: 0,+ sessionId: "remote-session",+ turnId: "turn-remote",+ type: "channel.delivery.completed",+ });+ await completeTurn(runtime.hooks, "remote-session", "turn-remote");+ });+ await runtime.provider.forceFlush();++ const spans = runtime.exporter.getFinishedSpans();+ const channelDelivery = byName(spans, "agent.channel.delivery")[0]!;+ const turn = byName(spans, "agent.turn")[0]!;+ expect(byName(spans, "agent.session")).toHaveLength(0);+ expect(channelDelivery.parentSpanContext).toMatchObject(parentTraceContext);+ expect(turn.parentSpanContext).toMatchObject(parentTraceContext);+ expect(turn.spanContext().traceId).toBe(parentTraceContext.traceId);+ });++ it("moves a delivery into the same rolled window as its resulting turn", async () => {+ const stateStore = new InMemoryAgentTraceStateStore();+ const runtime = createRuntime(stateStore);+ const ctx = new ContextContainer();+ const delivery = {+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-roll",+ };+ const idempotencyKey = `channel-delivery:session-1:${delivery.deliveryId}`;++ await contextStorage.run(ctx, async () => {+ await runtime.hooks.publish({+ agentName: "support",+ channelKind: "channel:slack",+ idempotencyKey: sessionIdempotencyKey("session-1"),+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "session.started",+ });+ const session = stateStore.getSession("session-1")!;+ stateStore.setSession("session-1", {+ ...session,+ turnsInWindow: SESSION_WINDOW_TURN_LIMIT,+ });+ await runtime.hooks.publish({+ delivery,+ idempotencyKey,+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "channel.delivery.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: turnIdempotencyKey("session-1", "turn-roll"),+ rootSessionId: "session-1",+ sequence: SESSION_WINDOW_TURN_LIMIT,+ sessionId: "session-1",+ turnId: "turn-roll",+ type: "turn.started",+ });+ await runtime.hooks.publish({+ delivery,+ idempotencyKey,+ outcome: "completed",+ rootSessionId: "session-1",+ sequence: SESSION_WINDOW_TURN_LIMIT,+ sessionId: "session-1",+ turnId: "turn-roll",+ type: "channel.delivery.completed",+ });+ await completeTurn(runtime.hooks, "session-1", "turn-roll");+ });+ await runtime.provider.forceFlush();++ const spans = runtime.exporter.getFinishedSpans();+ const rolledSession = byName(spans, "agent.session").find(+ (span) => span.attributes["agent.session.window"] === 1,+ )!;+ const channelDelivery = byName(spans, "agent.channel.delivery")[0]!;+ const turn = byName(spans, "agent.turn")[0]!;+ expect(channelDelivery.parentSpanContext?.spanId).toBe(rolledSession.spanContext().spanId);+ expect(channelDelivery.spanContext().traceId).toBe(turn.spanContext().traceId);+ expect(channelDelivery.attributes["agent.session.window"]).toBe(1);+ });+it("emits the agent hierarchy in one session trace", async () => {const runtime = createRuntime();const delivery = runtime.tracer.startSpan("workflow.delivery");diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex bbbbe2663..59699511a 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -27,6 +27,7 @@ import {import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";import { createAgentActionInstrumentation } from "#tracing/agent-action-instrumentation.js";import { createAgentApprovalInstrumentation } from "#tracing/agent-approval-instrumentation.js";+import { createAgentChannelDeliveryInstrumentation } from "#tracing/agent-channel-delivery-instrumentation.js";import { createAgentToolInstrumentation } from "#tracing/agent-tool-instrumentation.js";import { setAgentUsage } from "#tracing/agent-otel-usage.js";import type {@@ -478,6 +479,15 @@ export function createAgentOtelInstrumentation(return state;};+ const channelDeliveries = createAgentChannelDeliveryInstrumentation({+ ensureSessionContext,+ frameworkVersion: input.frameworkVersion,+ idGenerator: input.idGenerator,+ recordInputs,+ stateStore: input.stateStore,+ tracer: input.tracer,+ });+const advanceSessionWindow = (sessionId: string,session: AgentSessionTraceState,@@ -518,6 +528,7 @@ export function createAgentOtelInstrumentation(// what stops the projection from being built upstream.capture: recordInputs || recordOutputs ? "content" : "metadata",events: {+ ...channelDeliveries,"action.completed": actions.events["action.completed"],"action.failed": actions.events["action.failed"],async "action.started"(event, ctx) {diff --git a/packages/eve/src/tracing/content-attributes.test.ts b/packages/eve/src/tracing/content-attributes.test.tsindex edacb0268..2a9c68c93 100644--- a/packages/eve/src/tracing/content-attributes.test.ts+++ b/packages/eve/src/tracing/content-attributes.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";import { withoutDeclinedContent } from "#tracing/content-attributes.js";const ATTRIBUTES = {+ "agent.channel.delivery.input": '{"message":"private"}',"ai.prompt.messages": "what the user said","ai.response.finish_reason": "stop","ai.response.text": "what the model said",@@ -44,6 +45,7 @@ describe("withoutDeclinedContent", () => {expect(withoutDeclinedContent(ATTRIBUTES, { recordInputs: true, recordOutputs: false }),).toEqual({+ "agent.channel.delivery.input": '{"message":"private"}',"ai.prompt.messages": "what the user said","ai.response.finish_reason": "stop","gen_ai.request.model": "test-model",diff --git a/packages/eve/src/tracing/content-attributes.ts b/packages/eve/src/tracing/content-attributes.tsindex a31cdc4ed..5e9f33062 100644--- a/packages/eve/src/tracing/content-attributes.ts+++ b/packages/eve/src/tracing/content-attributes.ts@@ -15,6 +15,7 @@/** Prompts, instructions, tool arguments — what went in. */const INPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([+ "agent.channel.delivery.input","ai.documents","ai.prompt","ai.prompt.messages",