Skip to content

The open-source app everyone uses to manage agents at work

fix(observability): trace pre-dispatch controller waits (#16515)

Dotta0e58f29eb702
diff --git a/doc/observability.md b/doc/observability.md
index f50ba6c314..6068bc1302 100644
--- a/doc/observability.md
+++ b/doc/observability.md
@@ -242,6 +242,14 @@ preparation, execution and settlement spans. They are controller work, not model
turn time, and carry only the existing run hash. Nested phases overlap their
containing backend or lifecycle intervals; do not add both durations.
+Before provider dispatch, `execution.usage_startup` measures durable usage-recorder
+initialization. `execution.mcp_handoff` measures native MCP server resolution or
+legacy managed MCP configuration. `execution.dispatch_admission` measures the
+final admission gate, including its nested `execution.budget_reservation` span.
+The admission span ends after the provider promise is handed off, not after the
+provider turn finishes. These spans use only the existing run hash. They close
+gaps between preparation and execution; their nested intervals are not additive.
+
All preparation timing uses the endpoint gate, even when another library has
registered a tracer provider. Tracer lookup, callback, status or ending failures
cannot repeat work or replace its result or error. Automatic database children
diff --git a/server/src/services/execution-preparation-timing.test.ts b/server/src/services/execution-preparation-timing.test.ts
index f6467f5b54..17bda2f419 100644
--- a/server/src/services/execution-preparation-timing.test.ts
+++ b/server/src/services/execution-preparation-timing.test.ts
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
-import { createRunPreparationTiming } from "./execution-preparation-timing.js";
+import { createRunPreparationTiming, measureRunPhase, withRunExecutionTiming } from "./execution-preparation-timing.js";
const handles = vi.hoisted(() => ({ getTracer: vi.fn(), startActiveSpan: vi.fn() }));
vi.mock("@opentelemetry/api", () => ({
@@ -37,6 +37,25 @@ it.each(["lookup", "start", "after_callback", "missing_callback", "rejected", "e
it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async endpoint => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
handles.getTracer.mockClear();
- await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);
- expect(handles.getTracer).not.toHaveBeenCalled();
+ const warn = vi.spyOn(console, "warn");
+ const error = vi.spyOn(console, "error");
+ try {
+ await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);
+ for (const phase of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission", "execution.budget_reservation"] as const) {
+ const result = {};
+ const work = vi.fn(() => result);
+ await expect(withRunExecutionTiming("run", () => measureRunPhase(phase, work))).resolves.toBe(result);
+ expect(work).toHaveBeenCalledOnce();
+ const original = new Error("original startup failure");
+ const failed = vi.fn(() => { throw original; });
+ await expect(withRunExecutionTiming("run", () => measureRunPhase(phase, failed))).rejects.toBe(original);
+ expect(failed).toHaveBeenCalledOnce();
+ }
+ expect(handles.getTracer).not.toHaveBeenCalled();
+ expect(warn).not.toHaveBeenCalled();
+ expect(error).not.toHaveBeenCalled();
+ } finally {
+ warn.mockRestore();
+ error.mockRestore();
+ }
});
diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts
index 0df68c93e3..baa3858b33 100644
--- a/server/src/services/execution-preparation-timing.ts
+++ b/server/src/services/execution-preparation-timing.ts
@@ -74,6 +74,7 @@ export type ExecutionPhase =
| "execution.runtime" | "execution.workspace_ownership" | "execution.workspace_finalize" | "execution.workspace_restore" | "execution.workspace_commit"
| "execution.workspace_ownership_wait" | "execution.realtime"
| "execution.usage" | "execution.native_finalize"
+ | "execution.usage_startup" | "execution.mcp_handoff" | "execution.dispatch_admission" | "execution.budget_reservation"
| "completion.run" | "completion.output" | "completion.terminal_write" | "completion.cost" | "completion.liveness" | "completion.summary"
| "response.transaction" | "response.transaction.entry" | "response.transaction.commit" | "response.transaction.rollback" | "response.row_lock" | "response.write"
| "response.resolve" | "response.authorize" | "response.add_comment" | "response.activity" | "response.publish";
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 9d578682e4..8c0f5b32bf 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -2876,8 +2876,9 @@ export function heartbeatService(
) => {
// Admission can wait for accounting locks. Finish that wait before the
// final ownership gate, whose callback must enter the adapter directly.
- const reservation = await reserveRunBudget(db, run.companyId, run.id,
- readNonEmptyString(runLedgerScope.projectId), runLedgerScope, runOptions.nativeLeaseOwner);
+ const reservation = await measureRunPhase("execution.budget_reservation", () =>
+ reserveRunBudget(db, run.companyId, run.id,
+ readNonEmptyString(runLedgerScope.projectId), runLedgerScope, runOptions.nativeLeaseOwner));
let entered = false;
try {
return await dispatchResolvedInteractionContinuationAfterAdmission((markDispatchStarted) => {
diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts
index f124baeb71..361fe422b6 100644
--- a/server/src/services/heartbeat/runtime-execution.test.ts
+++ b/server/src/services/heartbeat/runtime-execution.test.ts
@@ -7,7 +7,7 @@ import { eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { trace } from "@opentelemetry/api";
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
-import { measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js";
+import { createRunPreparationTiming, measureRunPhase, measureWorkspaceSyncPhase, withRunExecutionTiming } from "../execution-preparation-timing.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import * as legacyRecovery from "../legacy-execution-recovery.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
@@ -246,6 +246,13 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
}]);
await provider.forceFlush();
const spans = exporter.getFinishedSpans();
+ const attempt = spans.find(span => span.name === "task.execution_attempt")!;
+ expect(attempt).toBeDefined();
+ for (const name of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission"]) {
+ const startup = spans.filter(span => span.name === name);
+ expect(startup).toHaveLength(1);
+ expect(startup[0].parentSpanContext?.spanId).toBe(attempt.spanContext().spanId);
+ }
const copyback = spans.filter(span => span.name === "execution.workspace_restore");
const commit = spans.filter(span => span.name === "execution.workspace_commit");
expect(copyback).toHaveLength(1);
@@ -260,6 +267,47 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
}
});
+ it.each([false, true])("ends admission timing without losing later provider spans (native: %s)", async native => {
+ const { input, adapter, run } = await fixture(native);
+ const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+ vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
+ const exporter = new InMemorySpanExporter();
+ const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] });
+ expect(trace.setGlobalTracerProvider(provider)).toBe(true);
+ let release!: (result: Awaited<ReturnType<typeof executePaperclipNativeSession>>) => void;
+ const providerResult = new Promise<Awaited<ReturnType<typeof executePaperclipNativeSession>>>(resolve => { release = resolve; });
+ const providerWork = async () => {
+ const result = await providerResult;
+ await createRunPreparationTiming(run.id, "native.controller")("coordinator_release", async () => {});
+ return result;
+ };
+ if (native) vi.mocked(executePaperclipNativeSession).mockImplementationOnce(providerWork);
+ else adapter.execute.mockImplementationOnce(providerWork);
+ const running = withRunExecutionTiming(run.id, () => executeHeartbeatRuntime(db, input));
+ try {
+ await vi.waitFor(() => {
+ expect(exporter.getFinishedSpans().filter(span => span.name === "execution.dispatch_admission")).toHaveLength(1);
+ }, { timeout: 5_000 });
+ expect(native ? executePaperclipNativeSession : adapter.execute).toHaveBeenCalledOnce();
+ expect(exporter.getFinishedSpans().find(span => span.name === "task.execution_attempt")).toBeUndefined();
+ expect(await finalizations()).toHaveLength(0);
+ release({ exitCode: 0, signal: null, timedOut: false });
+ await running;
+ const spans = exporter.getFinishedSpans();
+ const attempt = spans.find(span => span.name === "task.execution_attempt")!;
+ const later = spans.filter(span => span.name === "native.controller.coordinator_release");
+ expect(later).toHaveLength(1);
+ expect(later[0].spanContext().traceId).toBe(attempt.spanContext().traceId);
+ expect(later[0].parentSpanContext?.spanId).toBe(attempt.spanContext().spanId);
+ } finally {
+ release({ exitCode: 0, signal: null, timedOut: false });
+ await running;
+ vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
+ trace.disable();
+ await provider.shutdown();
+ }
+ });
+
it.each(["codex", "grok_bot", "openai_dot"] as const)("retains user files only when %s can access the workspace after a dispatch failure", async provider => {
const { input, run } = await fixture(true, provider);
input.workspace.nativeWorkspaceSync = {
diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts
index 95a9fa88be..c860d2a8e2 100644
--- a/server/src/services/heartbeat/runtime-execution.ts
+++ b/server/src/services/heartbeat/runtime-execution.ts
@@ -1,3 +1,4 @@
+import { AsyncLocalStorage } from "node:async_hooks";
import { measureRunPhase } from "../execution-preparation-timing.js";
import { parseNativeSessionGoalControl } from "./scheduling.js";
import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";
@@ -231,6 +232,10 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
const { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions;
const { providerTraceCapture, nativeRunnerPreparationSpans, attestedQuestionResponseAtMs, attemptStartedAtMs, environmentAcquireStartedAtMs, environmentRealizeEndedAtMs } = input.trace;
const { executionControl, executionPhaseContext, dispatchResolvedInteractionContinuationWithAtomicGate } = input.control;
+ const dispatchWithTiming: HeartbeatRuntimeExecutionInput["control"]["dispatchResolvedInteractionContinuationWithAtomicGate"] = dispatch => {
+ const providerDispatch = AsyncLocalStorage.bind(dispatch);
+ return measureRunPhase("execution.dispatch_admission", () => dispatchResolvedInteractionContinuationWithAtomicGate(providerDispatch));
+ };
const { identityRedactor, appendIdentityRedactedLog, onLog, onAdapterEvent, onAdapterMeta } = input.output;
const { executionWorkspacesSvc, instanceSettings, envOrchestrator, upsertTaskSession, getRun, cancelRunInternal, enqueueWakeup, persistRunProcessMetadata, recordCurrentHeartbeatRunRuntimeProgress, dispatchPendingNativeStatusWakeups } = input.services;
const localAgentJwtScope =
@@ -481,7 +486,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
adapterFinalizeOutcome = status;
};
- const usageRecorder = await createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType });
+ const usageRecorder = await measureRunPhase("execution.usage_startup", () =>
+ createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }));
input.effects.onUsageCaptureReady(usageRecorder.persistFailure);
let adapterResult: AdapterExecutionResult;
const runGoalControlRequestId = readNonEmptyString(
@@ -496,12 +502,12 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
nativeExecution.runtimeContext.mcp.bindingId
? nativeExecution.runtimeContext.mcp.digest
: null;
- const nativeMcpServers = "homeMode" in nativeExecution && nativeExecution.homeMode === "preconfigured" ? [] : await buildPaperclipRuntimeMcpServers({
+ const nativeMcpServers = "homeMode" in nativeExecution && nativeExecution.homeMode === "preconfigured" ? [] : await measureRunPhase("execution.mcp_handoff", () => buildPaperclipRuntimeMcpServers({
db,
agent,
runId: run.id,
expectedAssignmentDigest: expectedNativeMcpDigest,
- });
+ }));
if ("runtimeContext" in nativeExecution) {
if (nativeMcpServers.length > 1)
throw new Error(
@@ -564,7 +570,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
}),
);
const guardedDispatch =
- await dispatchResolvedInteractionContinuationWithAtomicGate(
+ await dispatchWithTiming(
(markDispatchStarted) => {
return executePaperclipNativeSession({
db,
@@ -772,19 +778,19 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
if (runtimeTools && runtimeToolDelivery === "invocation_context") {
adapterContext.paperclipRuntimeTools = runtimeTools;
}
- const managedMcpConfig = await createManagedMcpRunConfig({
+ const managedMcpConfig = await measureRunPhase("execution.mcp_handoff", () => createManagedMcpRunConfig({
db,
agent,
runId: run.id,
config: runtimeConfig,
projectId: issueRef?.projectId ?? null,
issueId: issueRef?.id ?? null,
- });
+ }));
if (managedMcpConfig) {
adapterContext.paperclipManagedMcp = managedMcpConfig;
}
const guardedDispatch =
- await dispatchResolvedInteractionContinuationWithAtomicGate(
+ await dispatchWithTiming(
(markDispatchStarted) => {
input.effects.onLegacyAdapterEntered();
return withAdapterExecutionPhase(executionPhaseContext, "adapter_execution", () => adapter.execute({