The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(observability): add provider OTel spans - cache-hit flag, plugin tracer, Daytona pack/transfer spans (#10764)
Nicky Leachaf6b32d82a30
diff --git a/packages/adapter-utils/src/acpx-engine/startup-timing.ts b/packages/adapter-utils/src/acpx-engine/startup-timing.tsindex ada1877fee..4bd2fbbc7f 100644--- a/packages/adapter-utils/src/acpx-engine/startup-timing.ts+++ b/packages/adapter-utils/src/acpx-engine/startup-timing.ts@@ -86,6 +86,8 @@ export const SANDBOX_STARTUP_SPAN_ATTRS = {execNetworkMs: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}exec.network_ms`,/** Whether one execution sits on the startup critical path. */execCriticalPath: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}exec.critical_path`,+ /** Whether the provider served the sandbox handle from its warm cache. */+ execCacheHit: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}exec.cache_hit`,/** The root-span wall time of the whole bring-up. */rootWallMs: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}root.wall_ms`,/** The sum of the step wall times of the whole bring-up. */@@ -108,6 +110,12 @@ export const SANDBOX_STARTUP_SPAN_ATTRS = {handshakeEnsureSessionWallMs: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}handshake.ensure_session.wall_ms`,/** A shared low-cardinality tag that marks two steps as one parallel batch. */batch: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}batch`,+ /** The host-local wall time of the pack step (build the tarball). */+ packWallMs: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}pack.wall_ms`,+ /** The wall time of the transfer step (upload the files to the sandbox). */+ transferWallMs: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}transfer.wall_ms`,+ /** The number of serial guard round trips before one transfer. */+ transferGuardCount: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}transfer.guard.count`,} as const;/** The closed value set for the `outcome` attribute. */diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.tsindex e8f5c5bb0a..b1e1bcbfe7 100644--- a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts+++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts@@ -11,9 +11,53 @@ import type {PluginSyncFileMapping,PluginSyncOperation,} from "@paperclipai/plugin-sdk";+import { getPluginTracer } from "./plugin.js";const execFileAsync = promisify(execFile);+// The span-attribute names. They mirror the host span-attribute contract by+// value. The plugin ships bundled, so it stays free of the host packages and+// repeats these strings; the host re-clamps a provider span by these exact keys.+const SPAN_ATTR_PREFIX = "paperclip.sandbox.startup.";+const SPAN_ATTR = {+ provider: `${SPAN_ATTR_PREFIX}provider`,+ packWallMs: `${SPAN_ATTR_PREFIX}pack.wall_ms`,+ transferWallMs: `${SPAN_ATTR_PREFIX}transfer.wall_ms`,+ transferGuardCount: `${SPAN_ATTR_PREFIX}transfer.guard.count`,+} as const;++/** The value of `SpanStatusCode.ERROR` in `@opentelemetry/api`. The plugin stays+ * OpenTelemetry-free, so it uses the numeric value directly. */+const SPAN_STATUS_CODE_ERROR = 2;++/**+ * Run one span-wrapped step through the plugin tracer. The pack step and the+ * transfer step share this helper. It seeds the provider family, runs the step,+ * sets the wall time, marks a thrown step failed, and always ends the span. The+ * tracer is a no-op until the host injects a live tracer, so the span never+ * changes the sync control flow.+ */+async function withProviderSpan<T>(input: {+ name: string;+ wallMsAttr: string;+ attributes?: Record<string, string | number | boolean>;+ run: () => Promise<T>;+}): Promise<T> {+ const span = getPluginTracer().startSpan(input.name, {+ attributes: { [SPAN_ATTR.provider]: "daytona", ...(input.attributes ?? {}) },+ });+ const startedAtMs = Date.now();+ try {+ return await input.run();+ } catch (error) {+ span.setStatus({ code: SPAN_STATUS_CODE_ERROR });+ throw error;+ } finally {+ span.setAttribute(input.wallMsAttr, Date.now() - startedAtMs);+ span.end();+ }+}+/** Convert a millisecond timeout to the whole-seconds value the Daytona SDK expects. */function toTimeoutSeconds(timeoutMs: number): number {return Math.max(1, Math.ceil(timeoutMs / 1000));@@ -404,9 +448,14 @@ async function syncInFileMappings(input: {bytesTransferred += (await fs.stat(mapping.sourcePath)).size;}+ // Count the serial guard round trips before the transfer, so the transfer span+ // records how much of the wall time is guard cost.+ let guardRoundTrips = 0;+// Ensure every target directory exists before the bulk upload writes its temp.const mkdirCommand = [...parentDirs].map((dir) => `mkdir -p ${shellQuote(dir)}`).join(" && ");await assertSandboxCommandOk(sandbox, mkdirCommand, timeoutSeconds, "syncIn mkdir");+ guardRoundTrips += 1;// Defense-in-depth beyond the lexical `assertConfinedSandboxPath`: a sandbox// can replace a target parent with a symlink to `/etc` so the string check@@ -419,13 +468,19 @@ async function syncInFileMappings(input: {timeoutSeconds,label: "inbound symlink-escape guard",});+ guardRoundTrips += 1;// A failed upload or a mid-batch `mv -f` failure leaves reserved temps (some// targets promoted, others not) — sweep every staged temp on any error so a// retry never accumulates stale `.paperclip-upload-*` scratch.try {// One batched bulk upload (single /files/bulk-upload) for all file mappings.- await sandbox.fs.uploadFiles(uploads, timeoutSeconds);+ await withProviderSpan({+ name: "transfer",+ wallMsAttr: SPAN_ATTR.transferWallMs,+ attributes: { [SPAN_ATTR.transferGuardCount]: guardRoundTrips },+ run: () => sandbox.fs.uploadFiles(uploads, timeoutSeconds),+ });// Apply the requested mode on the temp file BEFORE the rename so the target// never appears at a widened window — a secret lands `0600` at targetPath from@@ -490,16 +545,25 @@ async function syncInDirectoryMapping(input: {assertConfinedSandboxPath(remoteDir, mapping.targetPath, "target");return withHostTempDir(async (tmp) => {const archivePath = path.join(tmp, "sync-in.tar");- await createHostTarball({- localDir: mapping.sourcePath,- archivePath,- exclude: mapping.exclude,- followSymlinks: mapping.followSymlinks,+ // The pack step is host-local: it builds the tarball and makes no sandbox+ // round trip. The `pack` span records its wall time.+ await withProviderSpan({+ name: "pack",+ wallMsAttr: SPAN_ATTR.packWallMs,+ run: () => createHostTarball({+ localDir: mapping.sourcePath,+ archivePath,+ exclude: mapping.exclude,+ followSymlinks: mapping.followSymlinks,+ }),});const bytesTransferred = (await fs.stat(archivePath)).size;// The tar bytes ride the native bulk channel (string source ⇒ streamed);// only the extract/cleanup control commands use exec.const remoteTar = path.posix.join(remoteDir, scratchName(".tar"));+ // Count the serial guard round trips before the transfer, so the transfer+ // span records how much of the wall time is guard cost.+ let guardRoundTrips = 0;// Materialize the target dir first so the realpath guard resolves real// components, then confirm it (and any existing parent) canonicalizes inside// the remote dir — `tar -C` would otherwise follow a sandbox-planted symlink@@ -510,6 +574,7 @@ async function syncInDirectoryMapping(input: {timeoutSeconds,"syncIn mkdir",);+ guardRoundTrips += 1;await assertSandboxPathsConfined({sandbox,remoteDir,@@ -517,7 +582,14 @@ async function syncInDirectoryMapping(input: {timeoutSeconds,label: "inbound symlink-escape guard",});- await sandbox.fs.uploadFiles([{ source: archivePath, destination: remoteTar }], timeoutSeconds);+ guardRoundTrips += 1;+ await withProviderSpan({+ name: "transfer",+ wallMsAttr: SPAN_ATTR.transferWallMs,+ attributes: { [SPAN_ATTR.transferGuardCount]: guardRoundTrips },+ run: () =>+ sandbox.fs.uploadFiles([{ source: archivePath, destination: remoteTar }], timeoutSeconds),+ });// Bind validation and extraction into ONE sandbox invocation, then extract into// an OPEN directory inode rather than a path string. `exec 9<"$_pc_real"` itself// walks every ancestor of `$_pc_real` during the `open()` syscall, so a sandboxdiff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.tsindex 7f21b11b85..5cbc090ce2 100644--- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts@@ -33,8 +33,10 @@ import plugin, {setDaytonaHandleFreshnessClockForTest,__resetDaytonaSandboxHandleCacheForTest,__getDaytonaWritableDirsForTest,+ __setDaytonaPluginContextForTest,buildBwrapCommand,} from "./plugin.js";+import type { PluginContext } from "@paperclipai/plugin-sdk";import manifest from "./manifest.js";function createMockSandbox(overrides: {@@ -1157,6 +1159,41 @@ describe("Daytona sandbox provider plugin", () => {}});+ it("sets metadata.cacheHit false on a client.get miss and true on a warm-handle hit", async () => {+ process.env.DAYTONA_API_KEY = "host-key";+ const sandbox = createMockSandbox();+ sandbox.process.executeCommand.mockResolvedValue({+ exitCode: 0,+ result: "ok",+ artifacts: { stdout: "ok" },+ });+ mockGet.mockResolvedValue(sandbox);++ const execParams = {+ driverKey: "daytona" as const,+ companyId: "company-1",+ environmentId: "env-1",+ config: { timeoutMs: 300000, reuseLease: false },+ lease: { providerLeaseId: "sandbox-123", metadata: {} },+ command: "printf",+ args: ["hello"],+ cwd: "/workspace",+ timeoutMs: 1000,+ };++ // First execute: the handle cache is empty, so the lookup calls `client.get`+ // and reports a miss.+ const first = await plugin.definition.onEnvironmentExecute?.(execParams);+ expect(first!.metadata).toMatchObject({ cacheHit: false });+ expect(mockGet).toHaveBeenCalledTimes(1);++ // Second execute: the warm handle cache serves the handle, so the lookup+ // makes no `client.get` round trip and reports a hit.+ const second = await plugin.definition.onEnvironmentExecute?.(execParams);+ expect(second!.metadata).toMatchObject({ cacheHit: true });+ expect(mockGet).toHaveBeenCalledTimes(1);+ });+it("stages stdin in the sandbox filesystem when execution needs redirected input", async () => {process.env.DAYTONA_API_KEY = "host-key";const sandbox = createMockSandbox();@@ -2632,6 +2669,117 @@ describe("daytona native file-sync hooks", () => {});});+ // A recording tracer that captures every provider span the file sync opens.+ // It satisfies the structural plugin tracer contract.+ function createRecordingPluginTracer() {+ const spans: Array<{+ name: string;+ attributes: Record<string, unknown>;+ status: { code: number; message?: string } | null;+ ended: boolean;+ }> = [];+ const tracer = {+ startSpan(name: string, options?: { attributes?: Record<string, string | number | boolean> }) {+ const span = {+ name,+ attributes: { ...(options?.attributes ?? {}) } as Record<string, unknown>,+ status: null as { code: number; message?: string } | null,+ ended: false,+ setAttribute(key: string, value: unknown) {+ span.attributes[key] = value;+ },+ setStatus(status: { code: number; message?: string }) {+ span.status = status;+ },+ end() {+ span.ended = true;+ },+ };+ spans.push(span);+ return span;+ },+ };+ return { tracer, spans };+ }++ it("opens a transfer span with the guard round-trip count around the bulk file upload", async () => {+ const hostDir = await makeHostDir();+ const source = path.join(hostDir, "config.txt");+ await fs.writeFile(source, "plain");+ const sandbox = createMockSandbox();+ mockGet.mockResolvedValue(sandbox);++ const { tracer, spans } = createRecordingPluginTracer();+ const restore = __setDaytonaPluginContextForTest({ tracer } as unknown as PluginContext);+ try {+ await plugin.definition.onEnvironmentSyncIn?.({+ driverKey: "daytona",+ companyId: "company-1",+ environmentId: "env-1",+ config: { timeoutMs: 300000, reuseLease: false },+ lease: syncLease(),+ operations: [+ {+ operationId: "sync-op-1",+ files: [{ sourcePath: source, targetPath: `${REMOTE_DIR}/config.txt`, kind: "file" }],+ },+ ],+ });+ } finally {+ restore();+ }++ const transfer = spans.find((span) => span.name === "transfer");+ expect(transfer).toBeDefined();+ expect(transfer!.ended).toBe(true);+ // The two serial guard round trips before the transfer: mkdir + confinement.+ expect(transfer!.attributes["paperclip.sandbox.startup.transfer.guard.count"]).toBe(2);+ expect(transfer!.attributes["paperclip.sandbox.startup.provider"]).toBe("daytona");+ expect(typeof transfer!.attributes["paperclip.sandbox.startup.transfer.wall_ms"]).toBe("number");+ // A bulk file upload builds no host tarball, so it opens no pack span.+ expect(spans.find((span) => span.name === "pack")).toBeUndefined();+ });++ it("opens a pack span and a transfer span around a directory mapping sync", async () => {+ const hostDir = await makeHostDir();+ const sourceDir = path.join(hostDir, "assets");+ await fs.mkdir(sourceDir, { recursive: true });+ await fs.writeFile(path.join(sourceDir, "a.txt"), "alpha");+ const sandbox = createMockSandbox();+ mockGet.mockResolvedValue(sandbox);++ const { tracer, spans } = createRecordingPluginTracer();+ const restore = __setDaytonaPluginContextForTest({ tracer } as unknown as PluginContext);+ try {+ await plugin.definition.onEnvironmentSyncIn?.({+ driverKey: "daytona",+ companyId: "company-1",+ environmentId: "env-1",+ config: { timeoutMs: 300000, reuseLease: false },+ lease: syncLease(),+ operations: [+ {+ operationId: "sync-op-dir",+ files: [+ { sourcePath: sourceDir, targetPath: `${REMOTE_DIR}/.paperclip-runtime/assets`, kind: "directory" },+ ],+ },+ ],+ });+ } finally {+ restore();+ }++ const pack = spans.find((span) => span.name === "pack");+ expect(pack).toBeDefined();+ expect(pack!.ended).toBe(true);+ expect(typeof pack!.attributes["paperclip.sandbox.startup.pack.wall_ms"]).toBe("number");++ const transfer = spans.find((span) => span.name === "transfer");+ expect(transfer).toBeDefined();+ expect(transfer!.attributes["paperclip.sandbox.startup.transfer.guard.count"]).toBe(2);+ });+it("syncIn tars a directory mapping host-side honoring excludes and the followSymlinks flag, then extracts it in-sandbox via a single quoted tar command", async () => {const hostDir = await makeHostDir();const sourceDir = path.join(hostDir, "assets");diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.tsindex d12dfff0fa..cb2778c74b 100644--- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts@@ -9,8 +9,10 @@ import type {Resources,Sandbox,} from "@daytonaio/sdk";-import { definePlugin } from "@paperclipai/plugin-sdk";+import { definePlugin, NOOP_PLUGIN_TRACER } from "@paperclipai/plugin-sdk";import type {+ PluginContext,+ PluginTracer,PluginEnvironmentAcquireLeaseParams,PluginEnvironmentCancelInteractiveSetupParams,PluginEnvironmentCancelInteractiveSetupResult,@@ -46,6 +48,33 @@ import { performSyncIn, performSyncOut } from "./file-sync.js";// are deterministic. The timing path never calls `Date.now()` directly.let timingNow: () => number = () => Date.now();+// The plugin context, hoisted to a module variable in `setup(ctx)`. The+// lifecycle hooks and the file-sync helpers have no closure over `ctx`, so they+// read the tracer through `getPluginTracer()`. Before `setup` runs (or in a+// test) the tracer is a no-op, so a span never throws.+let pluginContext: PluginContext | null = null;++/**+ * Return the plugin tracer. It is the injected `ctx.tracer` after `setup`, or a+ * no-op before it. A provider span opened through it records only when tracing+ * is on and an active host trace context is present.+ */+export function getPluginTracer(): PluginTracer {+ return pluginContext?.tracer ?? NOOP_PLUGIN_TRACER;+}++/**+ * Test seam: set the module-level plugin context, and return a restore function.+ * `plugin.test.ts` uses it to inject a recording tracer without running `setup`.+ */+export function __setDaytonaPluginContextForTest(ctx: PluginContext | null): () => void {+ const previous = pluginContext;+ pluginContext = ctx;+ return () => {+ pluginContext = previous;+ };+}+/*** Test seam: override the provider-timing clock and return a restore function.* Not used in production, where the default wall clock always applies.@@ -998,6 +1027,11 @@ type SandboxHandleCacheEntry = {type SandboxLookupOptions = {bypassTeardownGate?: boolean;+ // Report the cache decision at the handle lookup. `true` means the warm+ // handle cache served the handle; `false` means the lookup called+ // `client.get`. The caller uses this to set the explicit exec `cache_hit`+ // flag, instead of the old `providerGetMs == 0` proxy.+ onCacheDecision?: (cacheHit: boolean) => void;};type SandboxHandleTeardownGate = {@@ -1182,6 +1216,9 @@ const sandboxHandleCache = (() => {const entry = entries.get(key);if (entry) {+ // The warm handle cache holds an entry, so this lookup serves the handle+ // without a `client.get` round trip. Report the cache decision now.+ options.onCacheDecision?.(true);const sandbox = await entry.sandbox;// Re-assert on every hit; evict + fail closed on any mismatch (C2).try {@@ -1205,6 +1242,9 @@ const sandboxHandleCache = (() => {}return sandbox;}+ // The warm handle cache holds no entry, so this lookup calls `client.get`.+ // Report the cache decision now, before the single-flight populate.+ options.onCacheDecision?.(false);// Single-flight: the first miss stores the in-flight promise under the// composite key so concurrent misses on the same lease share one `client.get`// instead of double-fetching. The promise lives only under this key (C5).@@ -1496,6 +1536,9 @@ async function executeOneShot(const plugin = definePlugin({async setup(ctx) {+ // Hoist the context to a module variable so the lifecycle hooks and the+ // file-sync helpers can read `ctx.tracer` — they have no closure over `ctx`.+ pluginContext = ctx;ctx.logger.info("Daytona sandbox provider plugin ready");},@@ -2119,13 +2162,24 @@ const plugin = definePlugin({// is a no-op for an already-started sandbox, so it is excluded from the get// measurement.const getStart = timingNow();+ // Decide the explicit `cache_hit` flag at the true cache decision: the+ // handle lookup reports whether the warm cache served the handle or the+ // lookup called `client.get`. This replaces the old `providerGetMs == 0`+ // proxy. The default `false` covers the theoretical case where the lookup+ // reports nothing.+ let cacheHit = false;const sandbox = await getSandbox({driverKey: params.driverKey,companyId: params.companyId,environmentId: params.environmentId,providerLeaseId,config,- }, { bypassTeardownGate: true });+ }, {+ bypassTeardownGate: true,+ onCacheDecision: (hit) => {+ cacheHit = hit;+ },+ });const getDurationMs = timingNow() - getStart;await ensureSandboxStarted(sandbox, toTimeoutSeconds(resolveTimeoutMs(params.timeoutMs, config)));// Read the advisory bwrap flags from the lease metadata and read the@@ -2149,7 +2203,7 @@ const plugin = definePlugin({}return {...result,- metadata: { ...(result.metadata ?? {}), getDurationMs },+ metadata: { ...(result.metadata ?? {}), getDurationMs, cacheHit },};});},diff --git a/packages/plugins/sdk/src/host-client-factory.ts b/packages/plugins/sdk/src/host-client-factory.tsindex 00f5d6efe3..f0c4db2de7 100644--- a/packages/plugins/sdk/src/host-client-factory.ts+++ b/packages/plugins/sdk/src/host-client-factory.ts@@ -182,6 +182,14 @@ export interface HostServices {log(params: WorkerToHostMethods["log"][0]): Promise<void>;};+ /** Provides `span.record`. The context carries the host-minted `traceparent`. */+ tracer: {+ record(+ params: WorkerToHostMethods["span.record"][0],+ context?: WorkerHostCallContext,+ ): Promise<void>;+ };+/** Provides `companies.list`, `companies.get`. */companies: {list(params: WorkerToHostMethods["companies.list"][0]): Promise<WorkerToHostMethods["companies.list"][1]>;@@ -416,6 +424,10 @@ const METHOD_CAPABILITY_MAP: Record<WorkerToHostMethodName, PluginCapability | n// Logger — always allowed"log": null,+ // Provider span sink — only a plugin that registers environment drivers may+ // emit a provider span. The gate rejects a span from any other plugin.+ "span.record": "environment.drivers.register",+// Companies"companies.list": "companies.read","companies.get": "companies.read",@@ -788,6 +800,11 @@ export function createHostClientHandlers(return services.logger.log(params);}),+ // Provider span sink. The context carries the host-minted `traceparent`.+ "span.record": gated("span.record", async (params, context) => {+ return services.tracer.record(params, context);+ }),+// Companies"companies.list": gated("companies.list", async (params, context) => {const rows = await services.companies.list(params);diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.tsindex d601080095..c8f8f4d406 100644--- a/packages/plugins/sdk/src/index.ts+++ b/packages/plugins/sdk/src/index.ts@@ -282,8 +282,13 @@ export type {PluginMetricsClient,PluginTelemetryClient,PluginLogger,+ PluginTracer,+ PluginSpan,} from "./types.js";+// Tracer no-op default (a value, so it re-exports here, not in the type block).+export { NOOP_PLUGIN_TRACER, NOOP_PLUGIN_SPAN } from "./types.js";+// Supporting types for context clientsexport type {ScopeKey,diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.tsindex 91d1d58f66..83e22a16b5 100644--- a/packages/plugins/sdk/src/protocol.ts+++ b/packages/plugins/sdk/src/protocol.ts@@ -291,6 +291,14 @@ export interface PluginInvocationScope {export interface PluginInvocationContext {id: string;scope: PluginInvocationScope;+ /**+ * An optional W3C `traceparent` for the active host span. The host mints it+ * per call from the active startup span. The worker treats it as opaque: it+ * tags its provider span with it and never derives parentage from it. The host+ * mints the parentage from its own invocation record, so a worker can never+ * forge a parent.+ */+ traceparent?: string;}/**@@ -300,6 +308,12 @@ export interface PluginInvocationContext {export interface WorkerHostCallContext {invocationScope?: PluginInvocationScope | null;invalidInvocationScope?: boolean;+ /**+ * The W3C `traceparent` the host minted for the echoed invocation. The host+ * recovers it from its own invocation record, not from the worker, so a worker+ * can never forge a span parent. The span host handler validates and uses it.+ */+ traceparent?: string;}// ---------------------------------------------------------------------------@@ -1274,6 +1288,25 @@ export interface WorkerToHostMethods {result: void,];+ // Provider span sink. The worker sends a finished provider span; the host+ // re-clamps the label and the attributes at its trust boundary, mints the+ // parentage from its own invocation record, and records the span through the+ // real tracer. The worker never sends the parent `traceparent`; the host+ // recovers it from the echoed invocation id. The RPC is capability-gated.+ "span.record": [+ params: {+ /** The bounded span name (for example `pack` or `transfer`). The host+ * clamps it to a closed set, so a name never carries free-form data. */+ name: string;+ /** The span attributes. The host drops every key that is not on the closed+ * plugin-span allowlist and re-clamps each remaining value. */+ attributes?: Record<string, string | number | boolean>;+ /** The optional span status. */+ status?: { code: number; message?: string };+ },+ result: void,+ ];+// Companies (read)"companies.list": [params: { limit?: number; offset?: number },diff --git a/packages/plugins/sdk/src/testing.ts b/packages/plugins/sdk/src/testing.tsindex e3255c8058..6fba7f9426 100644--- a/packages/plugins/sdk/src/testing.ts+++ b/packages/plugins/sdk/src/testing.ts@@ -45,6 +45,7 @@ import type {PermissionKey,PrincipalType,} from "./types.js";+import { NOOP_PLUGIN_TRACER } from "./types.js";import type {PluginEnvironmentValidateConfigParams,PluginEnvironmentValidationResult,@@ -2502,6 +2503,7 @@ export function createTestHarness(options: TestHarnessOptions): TestHarness {logs.push({ level: "debug", message, meta });},},+ tracer: NOOP_PLUGIN_TRACER,};const harness: TestHarness = {diff --git a/packages/plugins/sdk/src/types.ts b/packages/plugins/sdk/src/types.tsindex 55e34cbddb..ed1daedbf4 100644--- a/packages/plugins/sdk/src/types.ts+++ b/packages/plugins/sdk/src/types.ts@@ -1021,6 +1021,58 @@ export interface PluginLogger {debug(message: string, meta?: Record<string, unknown>): void;}+// ---------------------------------------------------------------------------+// Plugin tracer+// ---------------------------------------------------------------------------++/**+ * `ctx.tracer` — a minimal, OpenTelemetry-free span contract. The plugin worker+ * builds a real span through this surface; the host records it through the real+ * tracer. The shape is a subset of the `@opentelemetry/api` `Span` shape, so the+ * plugin SDK never imports `@opentelemetry/api`.+ *+ * A span with no active host trace context is a no-op: it accepts the calls and+ * ends without an effect. So a lifecycle hook can always open a span, and the+ * span records nothing until tracing is on.+ */+export interface PluginSpan {+ /** Set one bounded attribute. The host re-clamps every attribute at its trust+ * boundary, so an out-of-allowlist attribute never reaches a recorded span. */+ setAttribute(key: string, value: string | number | boolean): void;+ /** Set the span status. The host maps it onto the recorded span. */+ setStatus(status: { code: number; message?: string }): void;+ /** End the span. The worker sends the span data to the host once, here. */+ end(): void;+}++/**+ * `ctx.tracer` — a minimal, OpenTelemetry-free tracer contract. The plugin uses+ * it the same way as `ctx.logger`. The default is a no-op that never throws, so+ * a plugin span changes nothing until the host injects a live tracer and an+ * active host trace context.+ */+export interface PluginTracer {+ /** Start one span. `options.attributes` seeds the span attributes. */+ startSpan(+ name: string,+ options?: { attributes?: Record<string, string | number | boolean> },+ ): PluginSpan;+}++/** A shared no-op span. It satisfies the span contract and does nothing, so a+ * plugin with no injected tracer changes no behavior. */+export const NOOP_PLUGIN_SPAN: PluginSpan = {+ setAttribute() {},+ setStatus() {},+ end() {},+};++/** The default tracer. It opens no real span, so a lifecycle hook that wraps+ * work in a span behaves exactly as before when no live tracer is injected. */+export const NOOP_PLUGIN_TRACER: PluginTracer = {+ startSpan: () => NOOP_PLUGIN_SPAN,+};+// ---------------------------------------------------------------------------// Plugin metrics// ---------------------------------------------------------------------------@@ -2053,4 +2105,8 @@ export interface PluginContext {/** Structured logger. Output is captured and surfaced in the plugin health dashboard. */logger: PluginLogger;++ /** Tracer for provider spans. The default is a no-op; the host records a span+ * only when tracing is on and an active host trace context is present. */+ tracer: PluginTracer;}diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.tsindex 72dcf9238a..12739cf3d8 100644--- a/packages/plugins/sdk/src/worker-rpc-host.ts+++ b/packages/plugins/sdk/src/worker-rpc-host.ts@@ -1392,6 +1392,46 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHostnotifyHost("log", { level: "debug", message, meta });},},++ tracer: {+ startSpan(+ name: string,+ options?: { attributes?: Record<string, string | number | boolean> },+ ) {+ // Read the active host trace context from the per-call invocation+ // channel. A `traceparent` means a host span is active, so this span+ // may record. No `traceparent` means tracing is off: the span is a+ // no-op, so a lifecycle hook can always wrap work in a span.+ const hasTraceContext = Boolean(invocationContextStorage.getStore()?.traceparent);+ const attributes: Record<string, string | number | boolean> = {+ ...(options?.attributes ?? {}),+ };+ let status: { code: number; message?: string } | undefined;+ let ended = false;+ return {+ setAttribute(key: string, value: string | number | boolean): void {+ attributes[key] = value;+ },+ setStatus(next: { code: number; message?: string }): void {+ status = next;+ },+ end(): void {+ if (ended) return;+ ended = true;+ if (!hasTraceContext) return;+ // Send the finished span to the host once. The host re-clamps the+ // name and the attributes, mints the parentage from its own+ // invocation record, and records the span through the real tracer.+ // Fire-and-forget: a span must never block or fail plugin work.+ void callHost("span.record", {+ name,+ attributes,+ ...(status ? { status } : {}),+ }).catch(() => undefined);+ },+ };+ },+ },};}diff --git a/packages/plugins/sdk/tests/worker-rpc-host.test.ts b/packages/plugins/sdk/tests/worker-rpc-host.test.tsindex 8b0c709550..e34ca4de3c 100644--- a/packages/plugins/sdk/tests/worker-rpc-host.test.ts+++ b/packages/plugins/sdk/tests/worker-rpc-host.test.ts@@ -484,3 +484,133 @@ describe("worker configChanged cross-tenant guard", () => {}});});++describe("worker provider tracer", () => {+ it("default plugin tracer is a no-op that starts and ends a span without throwing", async () => {+ const { NOOP_PLUGIN_TRACER } = await import("../src/types.js");+ const span = NOOP_PLUGIN_TRACER.startSpan("pack", { attributes: { a: 1 } });+ expect(() => {+ span.setAttribute("b", 2);+ span.setStatus({ code: 1 });+ span.end();+ }).not.toThrow();+ });++ // Drive a plugin data handler that opens a provider span, and capture the+ // worker→host traffic. The host injects a `traceparent` on the invocation, so+ // the worker must emit one `span.record` request that echoes the invocation id+ // and carries the span name and attributes.+ async function runSpanProbe(invocation: PluginInvocationContext) {+ const hostToWorker = new PassThrough();+ const workerToHost = new PassThrough();+ const hostReadline = createInterface({ input: workerToHost });+ const pending = new Map<string, (response: JsonRpcResponse) => void>();+ const spanRecords: Array<{ params: unknown; invocationId?: string }> = [];+ let nextRequestId = 1;++ const plugin = definePlugin({+ async setup(ctx) {+ ctx.data.register("probe", async () => {+ const span = ctx.tracer.startSpan("pack", {+ attributes: { "paperclip.sandbox.startup.pack.wall_ms": 12 },+ });+ span.setAttribute("paperclip.sandbox.startup.provider", "daytona");+ span.end();+ return { ok: true };+ });+ },+ });++ const worker = startWorkerRpcHost({ plugin, stdin: hostToWorker, stdout: workerToHost });++ function callWorker(method: string, params: unknown, inv?: PluginInvocationContext) {+ const id = `host-${nextRequestId++}`;+ const request = {+ ...createRequest(method, params, id),+ ...(inv ? { paperclipInvocation: inv } : {}),+ };+ const result = new Promise<unknown>((resolve, reject) => {+ pending.set(id, (response) => {+ if ("error" in response && response.error) {+ reject(new Error(response.error.message));+ return;+ }+ resolve((response as { result?: unknown }).result);+ });+ });+ hostToWorker.write(serializeMessage(request));+ return result;+ }++ hostReadline.on("line", (line) => {+ const message = parseMessage(line);+ if (isJsonRpcResponse(message)) {+ pending.get(String(message.id))?.(message);+ pending.delete(String(message.id));+ return;+ }+ if (!isJsonRpcRequest(message)) return;+ if (message.method === "span.record") {+ spanRecords.push({+ params: message.params,+ invocationId: (message as { paperclipInvocationId?: string }).paperclipInvocationId,+ });+ hostToWorker.write(serializeMessage(createSuccessResponse(message.id, null)));+ }+ });++ try {+ await callWorker("initialize", {+ manifest: {+ id: "paperclip.tracer-test",+ apiVersion: 1,+ version: "1.0.0",+ displayName: "Tracer test",+ description: "Tracer test",+ author: "Paperclip",+ categories: ["automation"],+ capabilities: ["environment.drivers.register"],+ entrypoints: { worker: "dist/worker.js" },+ },+ config: {},+ instanceInfo: { instanceId: "test", hostVersion: "0.0.0" },+ apiVersion: 1,+ });+ await callWorker("getData", { key: "probe", companyId: "company-a", params: {} }, invocation);+ // Let the fire-and-forget span.record flush.+ await new Promise((resolve) => setTimeout(resolve, 20));+ return spanRecords;+ } finally {+ worker.stop();+ hostReadline.close();+ hostToWorker.destroy();+ workerToHost.destroy();+ }+ }++ it("emits one span.record with the name and attributes when a host trace context is active", async () => {+ const spanRecords = await runSpanProbe({+ id: "invocation-a",+ scope: { companyId: "company-a" },+ traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",+ });+ expect(spanRecords).toHaveLength(1);+ const record = spanRecords[0]!;+ expect(record.invocationId).toBe("invocation-a");+ expect(record.params).toMatchObject({+ name: "pack",+ attributes: {+ "paperclip.sandbox.startup.pack.wall_ms": 12,+ "paperclip.sandbox.startup.provider": "daytona",+ },+ });+ });++ it("emits no span.record when the invocation carries no traceparent (tracing off)", async () => {+ const spanRecords = await runSpanProbe({+ id: "invocation-a",+ scope: { companyId: "company-a" },+ });+ expect(spanRecords).toHaveLength(0);+ });+});diff --git a/packages/shared/src/telemetry/README.md b/packages/shared/src/telemetry/README.mdindex b01540b661..b916542ae3 100644--- a/packages/shared/src/telemetry/README.md+++ b/packages/shared/src/telemetry/README.md@@ -159,11 +159,73 @@ numeric attribute when the provider does not report the value.| `paperclip.sandbox.startup.exec.sandbox_ms` | number | yes | The in-sandbox run time of the execution. || `paperclip.sandbox.startup.exec.network_ms` | number | yes | The transport time the host adds; `wall_ms − wait_before_ms − sandbox_ms`. || `paperclip.sandbox.startup.exec.critical_path` | boolean | no | Whether the execution sits on the startup critical path. |+| `paperclip.sandbox.startup.exec.cache_hit` | boolean | yes | Whether the provider served the sandbox handle from its warm cache. || `paperclip.sandbox.startup.outcome` | string | no | The execution outcome (`ok` or `failed`). |+The plugin decides the cache hit at the sandbox-handle lookup. The span no+longer infers a cache hit from `wait_before_ms == 0`. Paperclip omits the+`cache_hit` attribute when the provider does not report the value.+To add a span attribute, extend the `SANDBOX_STARTUP_SPAN_ATTRS` allowlist inthe code first. Keep the attribute low-cardinality and free of user content.+### Provider spans++A sandbox provider plugin also opens spans for its own sync steps. These spans+use the `sandbox.provider.` name prefix. They share the+`paperclip.sandbox.startup.` attribute prefix and obey the same opt-in and+no-user-content rules as the startup spans above.++The plugin worker runs in a separate process from the host. So the host treats+every field of a worker-sent span as untrusted input. The host re-clamps the+span name and every attribute at one boundary, the `span.record` host handler,+before it records the span.++| Span | Scope | Parent |+| --- | --- | --- |+| `sandbox.provider.pack` | The host-local pack step that builds the upload tarball. It makes no sandbox round trip. | the active startup step span |+| `sandbox.provider.transfer` | The transfer step that uploads the files to the sandbox. | the active startup step span |+| `sandbox.provider.other` | Any span name outside the known set. | the active startup step span |++The host clamps the span name to the closed set `pack` and `transfer`. The host+maps a known name to `sandbox.provider.<name>`. The host maps any other value to+`sandbox.provider.other`, so a span name never carries free-form data.++The `sandbox.provider.*` spans use this closed attribute allowlist. The host+drops every other key, so a command, an argument, a path, an id, a standard+output, or a standard error never rides a provider span. The host records only+the attributes that the producer sends for one span.++| Attribute | Type | Optional | Meaning |+| --- | --- | --- | --- |+| `paperclip.sandbox.startup.provider` | string | no | The normalized provider family. |+| `paperclip.sandbox.startup.outcome` | string | yes | The step outcome (`ok`, `skipped`, or `failed`). |+| `paperclip.sandbox.startup.pack.wall_ms` | number | yes | The host-local wall time of the pack step. It rides the `sandbox.provider.pack` span. |+| `paperclip.sandbox.startup.transfer.wall_ms` | number | yes | The wall time of the transfer step. It rides the `sandbox.provider.transfer` span. |+| `paperclip.sandbox.startup.transfer.guard.count` | number | yes | The number of serial guard round trips before one transfer. It rides the `sandbox.provider.transfer` span. |++The `span.record` host handler enforces the allowlist. It re-maps `provider`+through the provider-family normalizer. It keeps `outcome` only when the value+is `ok`, `skipped`, or `failed`. It keeps a numeric attribute only when the+value is a finite number. It drops a status message and keeps only the numeric+status code. The handler never throws, because observability must not change the+sync control flow.++The `span.record` host method needs the `environment.drivers.register`+capability. So only a plugin that registers an environment driver may emit a+provider span. The capability gate rejects a provider span from any other+plugin.++The host parents each provider span to the active startup step span. The host+mints a W3C `traceparent` from the active step and passes it to the plugin+worker on the per-call invocation channel. The worker tags its span with the+`traceparent` and treats the value as opaque. The worker never derives the+parent from it. The host recovers the `traceparent` from its own invocation+record, so a worker can never forge a parent. The host validates the+`traceparent` and rejects a missing or malformed value. With no active host+trace context the worker sends no span, so the whole provider-span path is a+no-op.+## Dimension ValuesTelemetry dimension values must be primitives. Use only the value types alloweddiff --git a/server/src/__tests__/environment-execution-target.test.ts b/server/src/__tests__/environment-execution-target.test.tsindex a07cb68759..61eb5fba5b 100644--- a/server/src/__tests__/environment-execution-target.test.ts+++ b/server/src/__tests__/environment-execution-target.test.ts@@ -515,6 +515,7 @@ describe("resolveEnvironmentExecutionTarget", () => {A.execSandboxMs,A.execNetworkMs,A.execCriticalPath,+ A.execCacheHit,A.outcome,]);@@ -598,6 +599,50 @@ describe("resolveEnvironmentExecutionTarget", () => {expect(span.attributes[A.execWallMs] as number).toBeGreaterThanOrEqual(0);});+ it("carries the explicit exec cache_hit from result.metadata.cacheHit", async () => {+ const { tracer, spans } = createRecordingExecTracer();+ const runner = await runnerFor({+ provider: "daytona",+ execResult: {+ exitCode: 0,+ signal: null,+ timedOut: false,+ stdout: "ok",+ stderr: "",+ metadata: { durationMs: 600, getDurationMs: 0, cacheHit: true },+ },+ tracer,+ });++ await runner.execute({ command: "echo", args: ["a"] });++ const span = spans[0]!;+ // The flag comes from the metadata boolean, not from a zero handle-fetch.+ expect(span.attributes[A.execCacheHit]).toBe(true);+ });++ it("omits exec cache_hit when the provider reports no cacheHit metadata", async () => {+ const { tracer, spans } = createRecordingExecTracer();+ const runner = await runnerFor({+ provider: "daytona",+ execResult: {+ exitCode: 0,+ signal: null,+ timedOut: false,+ stdout: "ok",+ stderr: "",+ metadata: { durationMs: 600, getDurationMs: 15 },+ },+ tracer,+ });++ await runner.execute({ command: "echo", args: ["a"] });++ const span = spans[0]!;+ // A provider that omits the boolean yields no attribute — never `false`.+ expect(span.attributes).not.toHaveProperty(A.execCacheHit);+ });+it("omits each duration attribute when a provider returns no timing (does not throw, keeps provider)", async () => {const { tracer, spans } = createRecordingExecTracer();const runner = await runnerFor({diff --git a/server/src/__tests__/plugin-host-services-span.test.ts b/server/src/__tests__/plugin-host-services-span.test.tsnew file mode 100644index 0000000000..ea9e688e24--- /dev/null+++ b/server/src/__tests__/plugin-host-services-span.test.ts@@ -0,0 +1,199 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import { createHostClientHandlers } from "../../../packages/plugins/sdk/src/host-client-factory.js";+import type { WorkerHostCallContext } from "../../../packages/plugins/sdk/src/protocol.js";+import { SANDBOX_STARTUP_SPAN_ATTRS as A } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";+import {+ buildHostServices,+ clampProviderSpanAttributes,+ parseTraceparent,+} from "../services/plugin-host-services.js";++// Capture every span the host trust boundary hands to the real tracer.+const mockRecordSpan = vi.hoisted(() => vi.fn());++vi.mock("../instrumentation.js", () => ({+ recordProviderPluginSpan: mockRecordSpan,+ traceparentFromContextToken: () => undefined,+}));++function createEventBusStub() {+ return {+ forPlugin() {+ return { emit: vi.fn(), subscribe: vi.fn() };+ },+ } as never;+}++// A well-formed W3C traceparent (the host mints it; the handler validates it).+const VALID_TRACEPARENT = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01";++function servicesFor() {+ return buildHostServices({} as never, "plugin-record-id", "daytona", createEventBusStub());+}++function handlersFor(capabilities: readonly string[]) {+ return createHostClientHandlers({+ pluginId: "daytona",+ capabilities: capabilities as never,+ services: servicesFor(),+ });+}++describe("plugin provider span host handler", () => {+ beforeEach(() => {+ mockRecordSpan.mockReset();+ });++ it("records a span with the clamped name and the allowlisted attributes", async () => {+ const services = servicesFor();+ await services.tracer.record(+ {+ name: "pack",+ attributes: {+ [A.provider]: "daytona",+ [A.packWallMs]: 12,+ },+ },+ { traceparent: VALID_TRACEPARENT } as WorkerHostCallContext,+ );++ expect(mockRecordSpan).toHaveBeenCalledTimes(1);+ const call = mockRecordSpan.mock.calls[0]![0] as {+ name: string;+ parent: { traceId: string; spanId: string; traceFlags: number };+ attributes: Record<string, unknown>;+ };+ expect(call.name).toBe("sandbox.provider.pack");+ expect(call.parent).toEqual({+ traceId: "0af7651916cd43dd8448eb211c80319c",+ spanId: "b7ad6b7169203331",+ traceFlags: 1,+ });+ expect(call.attributes[A.provider]).toBe("daytona");+ expect(call.attributes[A.packWallMs]).toBe(12);+ });++ it("drops every forbidden attribute before the span reaches the tracer", async () => {+ const services = servicesFor();+ await services.tracer.record(+ {+ name: "transfer",+ attributes: {+ [A.transferGuardCount]: 2,+ // Forbidden fields that must never ride a span.+ [A.execCommand]: "bash",+ command: "rm -rf /",+ args: "--force",+ stdout: "secret output",+ stderr: "secret error",+ path: "/etc/passwd",+ extra: "leak",+ },+ },+ { traceparent: VALID_TRACEPARENT } as WorkerHostCallContext,+ );++ expect(mockRecordSpan).toHaveBeenCalledTimes(1);+ const attributes = (mockRecordSpan.mock.calls[0]![0] as { attributes: Record<string, unknown> })+ .attributes;+ // Only the allowlisted key survives; every forbidden key is dropped.+ expect(attributes).toEqual({ [A.transferGuardCount]: 2 });+ for (const forbidden of [A.execCommand, "command", "args", "stdout", "stderr", "path", "extra"]) {+ expect(attributes).not.toHaveProperty(forbidden);+ }+ });++ it("drops a status message (it could carry standard-stream text)", async () => {+ const services = servicesFor();+ await services.tracer.record(+ { name: "transfer", status: { code: 2, message: "secret error text" } },+ { traceparent: VALID_TRACEPARENT } as WorkerHostCallContext,+ );+ const call = mockRecordSpan.mock.calls[0]![0] as { status?: { code: number; message?: string } };+ expect(call.status).toEqual({ code: 2 });+ expect(call.status).not.toHaveProperty("message");+ });++ it("clamps an unknown span name to sandbox.provider.other", async () => {+ const services = servicesFor();+ await services.tracer.record(+ { name: "rm -rf / --no-preserve-root" },+ { traceparent: VALID_TRACEPARENT } as WorkerHostCallContext,+ );+ expect((mockRecordSpan.mock.calls[0]![0] as { name: string }).name).toBe(+ "sandbox.provider.other",+ );+ });++ it("rejects a malformed traceparent — no span is recorded", async () => {+ const services = servicesFor();+ for (const bad of [+ undefined,+ "not-a-traceparent",+ "00-xyz-b7ad6b7169203331-01",+ "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331", // missing flags+ "00-00000000000000000000000000000000-b7ad6b7169203331-01", // all-zero trace id+ ]) {+ await services.tracer.record(+ { name: "pack" },+ { traceparent: bad } as WorkerHostCallContext,+ );+ }+ expect(mockRecordSpan).not.toHaveBeenCalled();+ });++ it("rejects a span from a plugin that lacks the environment-driver capability", async () => {+ const handlers = handlersFor([]);+ await expect(+ handlers["span.record"](+ { name: "pack" },+ { traceparent: VALID_TRACEPARENT } as WorkerHostCallContext,+ ),+ ).rejects.toThrow(/capabilit/i);+ expect(mockRecordSpan).not.toHaveBeenCalled();+ });++ it("admits a span from a plugin that holds the environment-driver capability", async () => {+ const handlers = handlersFor(["environment.drivers.register"]);+ await handlers["span.record"](+ { name: "pack", attributes: { [A.provider]: "daytona" } },+ { traceparent: VALID_TRACEPARENT } as WorkerHostCallContext,+ );+ expect(mockRecordSpan).toHaveBeenCalledTimes(1);+ });+});++describe("parseTraceparent", () => {+ it("accepts a well-formed traceparent and returns the parts", () => {+ expect(parseTraceparent(VALID_TRACEPARENT)).toEqual({+ traceId: "0af7651916cd43dd8448eb211c80319c",+ spanId: "b7ad6b7169203331",+ traceFlags: 1,+ });+ });++ it("rejects malformed, all-zero, and forbidden-version traceparents", () => {+ expect(parseTraceparent(undefined)).toBeNull();+ expect(parseTraceparent("garbage")).toBeNull();+ expect(parseTraceparent("00-0af7651916cd43dd8448eb211c80319c-0000000000000000-01")).toBeNull();+ expect(parseTraceparent("ff-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01")).toBeNull();+ });+});++describe("clampProviderSpanAttributes", () => {+ it("keeps only allowlisted keys and normalizes the provider family", () => {+ expect(+ clampProviderSpanAttributes({+ [A.provider]: "some-operator-key",+ [A.packWallMs]: 7,+ [A.transferWallMs]: Number.NaN,+ [A.execCommand]: "bash",+ }),+ ).toEqual({+ // An unknown provider key maps to `plugin`, never the raw key.+ [A.provider]: "plugin",+ [A.packWallMs]: 7,+ // A non-finite number yields no attribute; `exec.command` is not allowed.+ });+ });+});diff --git a/server/src/instrumentation.ts b/server/src/instrumentation.tsindex b8f30c0cb3..246908ab00 100644--- a/server/src/instrumentation.ts+++ b/server/src/instrumentation.ts@@ -171,6 +171,86 @@ export function getStartupTraceContext(name = "paperclip.startup"): StartupTrace}}+/**+ * The parsed parts of a W3C `traceparent`. The host builds a remote parent span+ * context from these parts to parent a plugin span to the active host span.+ */+export interface ParsedTraceparent {+ traceId: string;+ spanId: string;+ traceFlags: number;+}++/**+ * Serialize an OTel context token to a W3C `traceparent` string. The host passes+ * the token to the plugin worker per call, so the worker's provider span can+ * parent to the active host span. The function reads the span context from the+ * token and formats it by hand, so it needs no registered propagator. It returns+ * `undefined` when `@opentelemetry/api` is absent, when the token holds no span+ * context, or when the span context is invalid.+ */+export function traceparentFromContextToken(contextToken: unknown): string | undefined {+ if (contextToken === undefined || contextToken === null) return undefined;+ try {+ const require = createRequire(import.meta.url);+ const api = require("@opentelemetry/api") as {+ trace?: { getSpanContext(context: unknown): { traceId: string; spanId: string; traceFlags: number } | undefined };+ };+ const spanContext = api.trace?.getSpanContext?.(contextToken);+ if (!spanContext) return undefined;+ const { traceId, spanId, traceFlags } = spanContext;+ if (!/^[0-9a-f]{32}$/.test(traceId) || !/^[0-9a-f]{16}$/.test(spanId)) return undefined;+ const flags = (traceFlags & 0xff).toString(16).padStart(2, "0");+ return `00-${traceId}-${spanId}-${flags}`;+ } catch {+ return undefined;+ }+}++/**+ * Record a plugin-originated provider span through the real tracer, parented to+ * a host span. The host handler validates and clamps the span data first (the+ * trust boundary), then passes the parsed parent and the clamped attributes+ * here. This function only does the OTel plumbing: it builds a remote parent+ * span context, opens the span, sets its attributes and status, and ends it. It+ * is a no-op when `@opentelemetry/api` is absent (the endpoint is unset) or when+ * the parent parts are invalid. It never throws — observability must not change+ * control flow.+ */+export function recordProviderPluginSpan(input: {+ name: string;+ parent: ParsedTraceparent;+ attributes: Record<string, string | number | boolean>;+ status?: { code: number; message?: string };+}): void {+ try {+ const require = createRequire(import.meta.url);+ const api = require("@opentelemetry/api") as {+ trace?: {+ getTracer(n: string): StartupTracerHandle;+ setSpanContext(context: unknown, spanContext: unknown): unknown;+ };+ context?: { active(): unknown };+ };+ const trace = api.trace;+ const context = api.context;+ if (!trace?.getTracer || !trace.setSpanContext || !context?.active) return;+ const remoteSpanContext = {+ traceId: input.parent.traceId,+ spanId: input.parent.spanId,+ traceFlags: input.parent.traceFlags,+ isRemote: true,+ };+ const parentContext = trace.setSpanContext(context.active(), remoteSpanContext);+ const tracer = trace.getTracer("paperclip.startup");+ const span = tracer.startSpan(input.name, { attributes: input.attributes }, parentContext);+ if (input.status) span.setStatus(input.status);+ span.end();+ } catch {+ // Observability must not change control flow.+ }+}+/*** Resolves once the OTel SDK has started (or once bootstrap has failed and* logged, or immediately when the feature is off). Await before constructingdiff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.tsindex 93611b0954..3c3e26aa9d 100644--- a/server/src/services/environment-execution-target.ts+++ b/server/src/services/environment-execution-target.ts@@ -62,6 +62,13 @@ function toFiniteNumber(value: unknown): number | undefined {return typeof value === "number" && Number.isFinite(value) ? value : undefined;}+/** Read a free-form metadata value as a boolean, or `undefined`. The provider+ * cache-hit flag rides the exec result's untyped `metadata`, so a provider that+ * omits or mistypes it yields no attribute — never a misleading `false`. */+function toBoolean(value: unknown): boolean | undefined {+ return typeof value === "boolean" ? value : undefined;+}+/*** The closed input for one `sandbox.exec` span. The seam builds it from the* exec result and the active step context. Every field is already bounded or@@ -82,6 +89,8 @@ interface SandboxExecSpanInput {sandboxMs: number | undefined;/** Whether the execution sits on the startup critical path. */criticalPath: boolean;+ /** Whether the provider served the sandbox handle from its warm cache. */+ cacheHit: boolean | undefined;}/**@@ -111,6 +120,13 @@ function setSandboxExecSpanAttributes(span: ExecSpan, input: SandboxExecSpanInpusetFiniteNumberAttr(span, A.execNetworkMs, input.wallMs - input.waitBeforeMs - input.sandboxMs);}span.setAttribute(A.execCriticalPath, input.criticalPath);+ // The explicit provider cache-hit flag, from `result.metadata.cacheHit`. The+ // plugin decides it at the handle lookup, so the span no longer infers a+ // cache hit from `wait_before_ms == 0`. A provider that omits it yields no+ // attribute.+ if (typeof input.cacheHit === "boolean") {+ span.setAttribute(A.execCacheHit, input.cacheHit);+ }const failed = input.exitCode !== 0;span.setAttribute(A.outcome,@@ -325,6 +341,7 @@ export async function resolveEnvironmentExecutionTarget(input: {waitBeforeMs: toFiniteNumber(result.metadata?.getDurationMs),sandboxMs: toFiniteNumber(result.metadata?.durationMs),criticalPath,+ cacheHit: toBoolean(result.metadata?.cacheHit),});} catch {// Observability must not change execution control flow.diff --git a/server/src/services/plugin-host-services.ts b/server/src/services/plugin-host-services.tsindex 0c2d10e2e2..a156c02bc7 100644--- a/server/src/services/plugin-host-services.ts+++ b/server/src/services/plugin-host-services.ts@@ -78,6 +78,12 @@ import { getTelemetryClient } from "../telemetry.js";import { accessService } from "./access.js";import { authorizationService, type AuthorizationActor } from "./authorization.js";import { redactEventPayload, sanitizeRecord } from "../redaction.js";+import type { WorkerHostCallContext } from "@paperclipai/plugin-sdk";+import {+ normalizeProviderFamily,+ SANDBOX_STARTUP_SPAN_ATTRS,+} from "@paperclipai/adapter-utils/acpx-engine/startup-timing";+import { recordProviderPluginSpan, type ParsedTraceparent } from "../instrumentation.js";// ---------------------------------------------------------------------------// SSRF protection for plugin HTTP fetch@@ -490,6 +496,124 @@ if (_logFlushInterval.unref) _logFlushInterval.unref();/** Maximum time (ms) to keep a session event subscription alive before forcing cleanup. */const SESSION_EVENT_SUBSCRIPTION_TIMEOUT_MS = 30 * 60 * 1_000; // 30 minutes+// ---------------------------------------------------------------------------+// Provider span trust boundary (the `span.record` host handler)+// ---------------------------------------------------------------------------+//+// The plugin worker runs in a separate process, so the host treats every field+// of a worker-sent span as untrusted input. The host re-clamps the span name+// and every attribute here, before it records the span. A worker-side or+// plugin-side helper is not sufficient; this is the single boundary.++const SPAN_ATTRS = SANDBOX_STARTUP_SPAN_ATTRS;++/** The closed set of provider span names a plugin may emit. */+const KNOWN_PROVIDER_SPAN_NAMES: ReadonlySet<string> = new Set(["pack", "transfer"]);++/** Clamp the span name to a closed, namespaced set. A known name maps to+ * `sandbox.provider.<name>`; any other value maps to `sandbox.provider.other`,+ * so a span name never carries free-form data. */+function clampProviderSpanName(raw: unknown): string {+ const name = typeof raw === "string" && KNOWN_PROVIDER_SPAN_NAMES.has(raw) ? raw : "other";+ return `sandbox.provider.${name}`;+}++/** The closed allowlist of attribute keys a provider span may carry. The host+ * drops every other key, so a command, an argument, a path, an id, a standard+ * output, a standard error, or an `extra` field can never ride a provider span. */+const PROVIDER_SPAN_ATTR_ALLOWLIST: ReadonlySet<string> = new Set<string>([+ SPAN_ATTRS.provider,+ SPAN_ATTRS.outcome,+ SPAN_ATTRS.packWallMs,+ SPAN_ATTRS.transferWallMs,+ SPAN_ATTRS.transferGuardCount,+]);++/** The subset of allowed keys that carry a finite number. */+const PROVIDER_SPAN_NUMERIC_ATTRS: ReadonlySet<string> = new Set<string>([+ SPAN_ATTRS.packWallMs,+ SPAN_ATTRS.transferWallMs,+ SPAN_ATTRS.transferGuardCount,+]);++/** The closed value set for the `outcome` attribute. */+const KNOWN_SPAN_OUTCOMES: ReadonlySet<string> = new Set(["ok", "skipped", "failed"]);++/**+ * Re-clamp the worker-sent attributes at the trust boundary. Drop every key that+ * is not on the allowlist. Re-map `provider` through `normalizeProviderFamily`,+ * bound `outcome` to its closed set, and keep a numeric attribute only when it+ * is a finite number. The result holds only bounded, low-cardinality values.+ */+export function clampProviderSpanAttributes(+ raw: Record<string, unknown> | undefined,+): Record<string, string | number | boolean> {+ const clamped: Record<string, string | number | boolean> = {};+ if (!raw) return clamped;+ for (const [key, value] of Object.entries(raw)) {+ if (!PROVIDER_SPAN_ATTR_ALLOWLIST.has(key)) continue;+ if (key === SPAN_ATTRS.provider) {+ clamped[key] = normalizeProviderFamily(typeof value === "string" ? value : undefined);+ continue;+ }+ if (key === SPAN_ATTRS.outcome) {+ if (typeof value === "string" && KNOWN_SPAN_OUTCOMES.has(value)) clamped[key] = value;+ continue;+ }+ if (PROVIDER_SPAN_NUMERIC_ATTRS.has(key)) {+ if (typeof value === "number" && Number.isFinite(value)) clamped[key] = value;+ continue;+ }+ }+ return clamped;+}++/**+ * Parse and validate a W3C `traceparent`. Return the parts, or `null` when the+ * value is absent or malformed. The host mints the value, but this is the trust+ * boundary, so it validates before use. It never logs the value.+ */+export function parseTraceparent(raw: string | undefined | null): ParsedTraceparent | null {+ if (typeof raw !== "string") return null;+ const match = /^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/.exec(raw);+ if (!match) return null;+ const [, version, traceId, spanId, flags] = match;+ if (version === "ff") return null; // the W3C spec forbids version 0xff+ if (traceId === "0".repeat(32)) return null; // an all-zero trace id is invalid+ if (spanId === "0".repeat(16)) return null; // an all-zero span id is invalid+ return { traceId, spanId, traceFlags: parseInt(flags, 16) };+}++/** Keep only the numeric status code. A status message could carry free-form+ * text, so the host drops it — never a standard-stream text on a span. */+function clampSpanStatus(+ status: { code?: unknown; message?: unknown } | undefined,+): { code: number } | undefined {+ if (!status || typeof status.code !== "number" || !Number.isFinite(status.code)) return undefined;+ return { code: status.code };+}++/**+ * Record a worker-sent provider span through the real tracer. This is the host+ * trust boundary: it validates the host-minted `traceparent`, re-clamps the span+ * name and every attribute, mints the parentage host-side, and drops a status+ * message. It rejects a span with a missing or malformed `traceparent`. It never+ * throws — observability must not change control flow.+ */+export function recordWorkerProviderSpan(+ params: { name: string; attributes?: Record<string, unknown>; status?: { code?: unknown; message?: unknown } },+ context: WorkerHostCallContext | undefined,+): void {+ const parent = parseTraceparent(context?.traceparent);+ if (!parent) return; // reject a missing or malformed traceparent+ recordProviderPluginSpan({+ name: clampProviderSpanName(params.name),+ parent,+ attributes: clampProviderSpanAttributes(params.attributes),+ ...(clampSpanStatus(params.status) ? { status: clampSpanStatus(params.status) } : {}),+ });+}+export function buildHostServices(db: Db,pluginId: string,@@ -1485,6 +1609,17 @@ export function buildHostServices(},},+ tracer: {+ async record(params, context) {+ // The host trust boundary: validate the host-minted `traceparent`,+ // re-clamp the span name and every attribute, mint the parentage+ // host-side, and record the span through the real tracer. The capability+ // gate in `createHostClientHandlers` already rejected an ungranted+ // plugin before this runs.+ recordWorkerProviderSpan(params, context);+ },+ },+companies: {async list(params) {return applyWindow((await companies.list()) as Company[], params);diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.tsindex af2c6b78ba..e0a5cbd0d8 100644--- a/server/src/services/plugin-worker-manager.ts+++ b/server/src/services/plugin-worker-manager.ts@@ -52,7 +52,9 @@ import type {WorkerToHostMethods,InitializeParams,} from "@paperclipai/plugin-sdk";+import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";import { logger } from "../middleware/logger.js";+import { traceparentFromContextToken } from "../instrumentation.js";// ---------------------------------------------------------------------------// Constants@@ -260,6 +262,10 @@ interface PendingRequest {interface ActiveInvocation {scope: PluginInvocationScope;timer?: ReturnType<typeof setTimeout>;+ // The host-minted W3C `traceparent` for the active startup span, or undefined+ // when no startup span is active. The span host handler reads it to mint the+ // parentage, so a worker never supplies the parent itself.+ traceparent?: string;}// ---------------------------------------------------------------------------@@ -621,11 +627,20 @@ export function createPluginWorkerHandle(}function registerInvocation(scope: PluginInvocationScope, ttlMs?: number): PluginInvocationContext {+ // Mint a W3C `traceparent` from the active startup span, so the worker's+ // provider span can parent to it. The host keeps the value on its own record+ // (below) and never trusts the worker to supply the parent. Outside a+ // measured startup step there is no active span, so this is undefined.+ const activeStep = getActiveStepContext();+ const traceparent = activeStep+ ? traceparentFromContextToken(activeStep.parentContext)+ : undefined;const invocation: PluginInvocationContext = {id: randomUUID(),scope,+ ...(traceparent ? { traceparent } : {}),};- const entry: ActiveInvocation = { scope };+ const entry: ActiveInvocation = { scope, traceparent };if (ttlMs !== undefined) {entry.timer = setTimeout(() => {activeInvocations.delete(invocation.id);@@ -703,7 +718,7 @@ export function createPluginWorkerHandle(}const entry = activeInvocations.get(invocationId);if (!entry) return { invalidInvocationScope: true };- return { invocationScope: entry.scope };+ return { invocationScope: entry.scope, traceparent: entry.traceparent };}/**