The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(observability): trace pre-dispatch controller waits (#16515)
Dotta0e58f29eb702
diff --git a/doc/observability.md b/doc/observability.mdindex f50ba6c314..6068bc1302 100644--- a/doc/observability.md+++ b/doc/observability.md@@ -242,6 +242,14 @@ preparation, execution and settlement spans. They are controller work, not modelturn time, and carry only the existing run hash. Nested phases overlap theircontaining backend or lifecycle intervals; do not add both durations.+Before provider dispatch, `execution.usage_startup` measures durable usage-recorder+initialization. `execution.mcp_handoff` measures native MCP server resolution or+legacy managed MCP configuration. `execution.dispatch_admission` measures the+final admission gate, including its nested `execution.budget_reservation` span.+The admission span ends after the provider promise is handed off, not after the+provider turn finishes. These spans use only the existing run hash. They close+gaps between preparation and execution; their nested intervals are not additive.+All preparation timing uses the endpoint gate, even when another library hasregistered a tracer provider. Tracer lookup, callback, status or ending failurescannot repeat work or replace its result or error. Automatic database childrendiff --git a/server/src/services/execution-preparation-timing.test.ts b/server/src/services/execution-preparation-timing.test.tsindex f6467f5b54..17bda2f419 100644--- a/server/src/services/execution-preparation-timing.test.ts+++ b/server/src/services/execution-preparation-timing.test.ts@@ -1,5 +1,5 @@import { afterEach, beforeEach, expect, it, vi } from "vitest";-import { createRunPreparationTiming } from "./execution-preparation-timing.js";+import { createRunPreparationTiming, measureRunPhase, withRunExecutionTiming } from "./execution-preparation-timing.js";const handles = vi.hoisted(() => ({ getTracer: vi.fn(), startActiveSpan: vi.fn() }));vi.mock("@opentelemetry/api", () => ({@@ -37,6 +37,25 @@ it.each(["lookup", "start", "after_callback", "missing_callback", "rejected", "eit.each([undefined, "", " "])("does not consult tracing for endpoint %s", async endpoint => {vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);handles.getTracer.mockClear();- await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);- expect(handles.getTracer).not.toHaveBeenCalled();+ const warn = vi.spyOn(console, "warn");+ const error = vi.spyOn(console, "error");+ try {+ await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);+ for (const phase of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission", "execution.budget_reservation"] as const) {+ const result = {};+ const work = vi.fn(() => result);+ await expect(withRunExecutionTiming("run", () => measureRunPhase(phase, work))).resolves.toBe(result);+ expect(work).toHaveBeenCalledOnce();+ const original = new Error("original startup failure");+ const failed = vi.fn(() => { throw original; });+ await expect(withRunExecutionTiming("run", () => measureRunPhase(phase, failed))).rejects.toBe(original);+ expect(failed).toHaveBeenCalledOnce();+ }+ expect(handles.getTracer).not.toHaveBeenCalled();+ expect(warn).not.toHaveBeenCalled();+ expect(error).not.toHaveBeenCalled();+ } finally {+ warn.mockRestore();+ error.mockRestore();+ }});diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.tsindex 0df68c93e3..baa3858b33 100644--- a/server/src/services/execution-preparation-timing.ts+++ b/server/src/services/execution-preparation-timing.ts@@ -74,6 +74,7 @@ export type ExecutionPhase =| "execution.runtime" | "execution.workspace_ownership" | "execution.workspace_finalize" | "execution.workspace_restore" | "execution.workspace_commit"| "execution.workspace_ownership_wait" | "execution.realtime"| "execution.usage" | "execution.native_finalize"+ | "execution.usage_startup" | "execution.mcp_handoff" | "execution.dispatch_admission" | "execution.budget_reservation"| "completion.run" | "completion.output" | "completion.terminal_write" | "completion.cost" | "completion.liveness" | "completion.summary"| "response.transaction" | "response.transaction.entry" | "response.transaction.commit" | "response.transaction.rollback" | "response.row_lock" | "response.write"| "response.resolve" | "response.authorize" | "response.add_comment" | "response.activity" | "response.publish";diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 9d578682e4..8c0f5b32bf 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -2876,8 +2876,9 @@ export function heartbeatService() => {// Admission can wait for accounting locks. Finish that wait before the// final ownership gate, whose callback must enter the adapter directly.- const reservation = await reserveRunBudget(db, run.companyId, run.id,- readNonEmptyString(runLedgerScope.projectId), runLedgerScope, runOptions.nativeLeaseOwner);+ const reservation = await measureRunPhase("execution.budget_reservation", () =>+ reserveRunBudget(db, run.companyId, run.id,+ readNonEmptyString(runLedgerScope.projectId), runLedgerScope, runOptions.nativeLeaseOwner));let entered = false;try {return await dispatchResolvedInteractionContinuationAfterAdmission((markDispatchStarted) => {diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.tsindex f124baeb71..361fe422b6 100644--- a/server/src/services/heartbeat/runtime-execution.test.ts+++ b/server/src/services/heartbeat/runtime-execution.test.ts@@ -7,7 +7,7 @@ import { eq, sql } from "drizzle-orm";import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";import { trace } from "@opentelemetry/api";import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";-import { measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js";+import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js";import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";import * as legacyRecovery from "../legacy-execution-recovery.js";import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";@@ -246,6 +246,13 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =}]);await provider.forceFlush();const spans = exporter.getFinishedSpans();+ const attempt = spans.find(span => span.name === "task.execution_attempt")!;+ expect(attempt).toBeDefined();+ for (const name of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission"]) {+ const startup = spans.filter(span => span.name === name);+ expect(startup).toHaveLength(1);+ expect(startup[0].parentSpanContext?.spanId).toBe(attempt.spanContext().spanId);+ }const copyback = spans.filter(span => span.name === "execution.workspace_restore");const commit = spans.filter(span => span.name === "execution.workspace_commit");expect(copyback).toHaveLength(1);@@ -260,6 +267,47 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =}});+ it.each([false, true])("ends admission timing without losing later provider spans (native: %s)", async native => {+ const { input, adapter, run } = await fixture(native);+ const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;+ vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");+ const exporter = new InMemorySpanExporter();+ const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] });+ expect(trace.setGlobalTracerProvider(provider)).toBe(true);+ let release!: (result: Awaited<ReturnType<typeof executePaperclipNativeSession>>) => void;+ const providerResult = new Promise<Awaited<ReturnType<typeof executePaperclipNativeSession>>>(resolve => { release = resolve; });+ const providerWork = async () => {+ const result = await providerResult;+ await createRunPreparationTiming(run.id, "native.controller")("coordinator_release", async () => {});+ return result;+ };+ if (native) vi.mocked(executePaperclipNativeSession).mockImplementationOnce(providerWork);+ else adapter.execute.mockImplementationOnce(providerWork);+ const running = withRunExecutionTiming(run.id, () => executeHeartbeatRuntime(db, input));+ try {+ await vi.waitFor(() => {+ expect(exporter.getFinishedSpans().filter(span => span.name === "execution.dispatch_admission")).toHaveLength(1);+ }, { timeout: 5_000 });+ expect(native ? executePaperclipNativeSession : adapter.execute).toHaveBeenCalledOnce();+ expect(exporter.getFinishedSpans().find(span => span.name === "task.execution_attempt")).toBeUndefined();+ expect(await finalizations()).toHaveLength(0);+ release({ exitCode: 0, signal: null, timedOut: false });+ await running;+ const spans = exporter.getFinishedSpans();+ const attempt = spans.find(span => span.name === "task.execution_attempt")!;+ const later = spans.filter(span => span.name === "native.controller.coordinator_release");+ expect(later).toHaveLength(1);+ expect(later[0].spanContext().traceId).toBe(attempt.spanContext().traceId);+ expect(later[0].parentSpanContext?.spanId).toBe(attempt.spanContext().spanId);+ } finally {+ release({ exitCode: 0, signal: null, timedOut: false });+ await running;+ vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);+ trace.disable();+ await provider.shutdown();+ }+ });+it.each(["codex", "grok_bot", "openai_dot"] as const)("retains user files only when %s can access the workspace after a dispatch failure", async provider => {const { input, run } = await fixture(true, provider);input.workspace.nativeWorkspaceSync = {diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.tsindex 95a9fa88be..c860d2a8e2 100644--- a/server/src/services/heartbeat/runtime-execution.ts+++ b/server/src/services/heartbeat/runtime-execution.ts@@ -1,3 +1,4 @@+import { AsyncLocalStorage } from "node:async_hooks";import { measureRunPhase } from "../execution-preparation-timing.js";import { parseNativeSessionGoalControl } from "./scheduling.js";import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";@@ -231,6 +232,10 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExeconst { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions;const { providerTraceCapture, nativeRunnerPreparationSpans, attestedQuestionResponseAtMs, attemptStartedAtMs, environmentAcquireStartedAtMs, environmentRealizeEndedAtMs } = input.trace;const { executionControl, executionPhaseContext, dispatchResolvedInteractionContinuationWithAtomicGate } = input.control;+ const dispatchWithTiming: HeartbeatRuntimeExecutionInput["control"]["dispatchResolvedInteractionContinuationWithAtomicGate"] = dispatch => {+ const providerDispatch = AsyncLocalStorage.bind(dispatch);+ return measureRunPhase("execution.dispatch_admission", () => dispatchResolvedInteractionContinuationWithAtomicGate(providerDispatch));+ };const { identityRedactor, appendIdentityRedactedLog, onLog, onAdapterEvent, onAdapterMeta } = input.output;const { executionWorkspacesSvc, instanceSettings, envOrchestrator, upsertTaskSession, getRun, cancelRunInternal, enqueueWakeup, persistRunProcessMetadata, recordCurrentHeartbeatRunRuntimeProgress, dispatchPendingNativeStatusWakeups } = input.services;const localAgentJwtScope =@@ -481,7 +486,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExeadapterFinalizeOutcome = status;};- const usageRecorder = await createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType });+ const usageRecorder = await measureRunPhase("execution.usage_startup", () =>+ createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }));input.effects.onUsageCaptureReady(usageRecorder.persistFailure);let adapterResult: AdapterExecutionResult;const runGoalControlRequestId = readNonEmptyString(@@ -496,12 +502,12 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExenativeExecution.runtimeContext.mcp.bindingId? nativeExecution.runtimeContext.mcp.digest: null;- const nativeMcpServers = "homeMode" in nativeExecution && nativeExecution.homeMode === "preconfigured" ? [] : await buildPaperclipRuntimeMcpServers({+ const nativeMcpServers = "homeMode" in nativeExecution && nativeExecution.homeMode === "preconfigured" ? [] : await measureRunPhase("execution.mcp_handoff", () => buildPaperclipRuntimeMcpServers({db,agent,runId: run.id,expectedAssignmentDigest: expectedNativeMcpDigest,- });+ }));if ("runtimeContext" in nativeExecution) {if (nativeMcpServers.length > 1)throw new Error(@@ -564,7 +570,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe}),);const guardedDispatch =- await dispatchResolvedInteractionContinuationWithAtomicGate(+ await dispatchWithTiming((markDispatchStarted) => {return executePaperclipNativeSession({db,@@ -772,19 +778,19 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExeif (runtimeTools && runtimeToolDelivery === "invocation_context") {adapterContext.paperclipRuntimeTools = runtimeTools;}- const managedMcpConfig = await createManagedMcpRunConfig({+ const managedMcpConfig = await measureRunPhase("execution.mcp_handoff", () => createManagedMcpRunConfig({db,agent,runId: run.id,config: runtimeConfig,projectId: issueRef?.projectId ?? null,issueId: issueRef?.id ?? null,- });+ }));if (managedMcpConfig) {adapterContext.paperclipManagedMcp = managedMcpConfig;}const guardedDispatch =- await dispatchResolvedInteractionContinuationWithAtomicGate(+ await dispatchWithTiming((markDispatchStarted) => {input.effects.onLegacyAdapterEntered();return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({