Skip to content

The open-source app everyone uses to manage agents at work

fix: retain bounded workspace transfer failure evidence (#15637)

Devin Foley1c4ce44e137b
diff --git a/doc/observability.md b/doc/observability.md
index f71a6a2b30..e8d6cbb215 100644
--- a/doc/observability.md
+++ b/doc/observability.md
@@ -661,6 +661,24 @@ The envelope excludes provider messages, response bodies, names, paths, and
credentials. The existing RPC error message and code remain unchanged, as do
restore classification, retries, and source retention. Older workers without the
envelope still report `unknown` when no structured cause is available.
+For native transfers, optional `workspaceRestoreTransferStep` identifies
+`sandbox_access`, `sandbox_guard`, `file_download`, `file_finalize`,
+`archive_create`, `archive_download`, `archive_validate`, or `archive_extract`.
+`workspaceRestoreTransferFailureKind` distinguishes a producer-confirmed
+`command_failed`, `download_failed`, `download_missing`, `unsafe_archive`,
+`listing_timeout`, `listing_entry_limit`, `listing_byte_limit`,
+`listing_line_limit`, or `listing_stderr_limit`. These labels are fixed values,
+not classifications inferred from provider message text. Missing labels remain
+absent. A listing deadline records `ETIMEDOUT`; sandbox and listing commands
+retain bounded nonzero exit codes even when their error text is wrapped.
+`workspaceRestoreRpcCode` retains only known numeric plugin RPC error codes from
+a typed error at the host's sync-out boundary. In particular, RPC `TIMEOUT`
+(`-32003`) is separate from the model execution's `timedOut` value; successful
+model execution does not exclude a later transfer timeout. Transfer evidence is
+isolated per outbound request and per restore settlement, then independently
+revalidated before persistence and Sentry projection. These fields only accompany
+`workspace_transfer` or `asset_restore` diagnostics and do not authorize cleanup
+or retries. Older workers remain compatible.
For `git_integration`, optional `workspaceRestoreGitCommand` identifies the fixed
command family: `rev_parse`, `symbolic_ref`, `merge_base`, `merge_tree`,
`commit_tree`, `update_ref`, or `log`. `workspaceRestoreGitFailureKind` is
diff --git a/packages/adapter-utils/src/workspace-restore-diagnostics.test.ts b/packages/adapter-utils/src/workspace-restore-diagnostics.test.ts
index 39e82e1929..936b68e302 100644
--- a/packages/adapter-utils/src/workspace-restore-diagnostics.test.ts
+++ b/packages/adapter-utils/src/workspace-restore-diagnostics.test.ts
@@ -1,7 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { classifyWorkspaceRestoreFailure } from "./workspace-restore-merge.js";
import {
- getWorkspaceRestoreDiagnostic, preserveWorkspaceRestoreErrorDiagnostic, recordWorkspaceRestoreDiagnostic,
+ withWorkspaceRestoreDiagnosticCapture, getWorkspaceRestoreDiagnostic, preserveWorkspaceRestoreErrorDiagnostic, recordWorkspaceRestoreDiagnostic,
sanitizeWorkspaceRestoreDiagnostic, withWorkspaceRestoreDiagnostics, withWorkspaceRestoreStep, withWorkspaceRestoreGitCommand,
type WorkspaceRestoreDiagnostic,
} from "./workspace-restore-diagnostics.js";
@@ -297,3 +297,33 @@ describe("Git integration diagnostic privacy and attribution", () => {
});
});
});
+
+
+it("isolates wrapped transfer evidence across settlements reusing a frozen error", async () => {
+ const shared = Object.freeze(new Error("private-message"));
+ let resume!: () => void;
+ const gate = new Promise<void>(resolve => { resume = resolve; });
+ let started!: () => void;
+ const ready = new Promise<void>(resolve => { started = resolve; });
+ const first = withWorkspaceRestoreDiagnosticCapture(async () => {
+ await expect(withWorkspaceRestoreDiagnostics("workspace", () => withWorkspaceRestoreStep("workspace_transfer", async () => {
+ preserveWorkspaceRestoreErrorDiagnostic(shared, {}, { transferStep: "archive_create" });
+ started(); await gate; throw shared;
+ }))).rejects.toBe(shared);
+ return getWorkspaceRestoreDiagnostic(shared);
+ });
+ await ready;
+ const second = await withWorkspaceRestoreDiagnosticCapture(async () => {
+ await expect(withWorkspaceRestoreDiagnostics("workspace", () => withWorkspaceRestoreStep("workspace_transfer", async () => {
+ throw preserveWorkspaceRestoreErrorDiagnostic(shared, {}, { transferStep: "archive_download" });
+ }))).rejects.toBe(shared);
+ return getWorkspaceRestoreDiagnostic(shared);
+ });
+ resume();
+ expect((await first)?.transferStep).toBe("archive_create");
+ expect(second?.transferStep).toBe("archive_download");
+ await withWorkspaceRestoreDiagnosticCapture(async () => {
+ await expect(withWorkspaceRestoreDiagnostics("workspace", () => withWorkspaceRestoreStep("workspace_transfer", async () => { throw shared; }))).rejects.toBe(shared);
+ expect(getWorkspaceRestoreDiagnostic(shared)?.transferStep).toBeUndefined();
+ });
+});
diff --git a/packages/adapter-utils/src/workspace-restore-diagnostics.ts b/packages/adapter-utils/src/workspace-restore-diagnostics.ts
index afb8d659c9..8ac378374f 100644
--- a/packages/adapter-utils/src/workspace-restore-diagnostics.ts
+++ b/packages/adapter-utils/src/workspace-restore-diagnostics.ts
@@ -15,6 +15,18 @@ const GIT_FAILURE_KINDS = new Set([
"merge_conflict", "invalid_object", "ref_conflict", "permission_denied", "unknown",
] as const);
type GitFailureKind = typeof GIT_FAILURE_KINDS extends Set<infer T> ? T : never;
+const TRANSFER_STEPS = new Set([
+ "sandbox_access", "sandbox_guard", "file_download", "file_finalize",
+ "archive_create", "archive_download", "archive_validate", "archive_extract",
+] as const);
+type TransferStep = typeof TRANSFER_STEPS extends Set<infer T> ? T : never;
+const TRANSFER_FAILURE_KINDS = new Set([
+ "command_failed", "download_failed", "download_missing", "unsafe_archive",
+ "listing_timeout", "listing_entry_limit", "listing_byte_limit", "listing_line_limit", "listing_stderr_limit",
+] as const);
+type TransferFailureKind = typeof TRANSFER_FAILURE_KINDS extends Set<infer T> ? T : never;
+// Deliberately revalidate the known plugin RPC constants at the persistence boundary.
+const RPC_CODES = new Set([-32000, -32001, -32002, -32003, -32004, -32005, -32006, -32099]);
export interface WorkspaceRestoreDiagnostic {
phase: RestorePhase;
step?: WorkspaceRestoreStep;
@@ -23,13 +35,16 @@ export interface WorkspaceRestoreDiagnostic {
exitCode?: number;
gitCommand?: WorkspaceRestoreGitCommand;
gitFailureKind?: GitFailureKind;
+ transferStep?: TransferStep;
+ transferFailureKind?: TransferFailureKind;
+ rpcCode?: number;
}
const ERROR_CODES = new Set([
"ENOENT", "EACCES", "EPERM", "ENOSPC", "EIO", "EXDEV", "ENOTDIR", "EISDIR",
"ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", "ENOTFOUND", "EAI_AGAIN",
"ABORT_ERR", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET",
]);
-type ErrorDiagnostic = Pick<WorkspaceRestoreDiagnostic, "errorCode" | "httpStatus" | "exitCode">;
+type ErrorDiagnostic = Pick<WorkspaceRestoreDiagnostic, "errorCode" | "httpStatus" | "exitCode" | "transferStep" | "transferFailureKind" | "rpcCode">;
interface DiagnosticScope {
active: boolean;
sequence: number;
@@ -44,11 +59,14 @@ const activeDiagnostic = new AsyncLocalStorage<DiagnosticScope>();
// Never attach a raw cause to an error just to retain a numeric Git exit code.
const wrappedDiagnostics = new WeakMap<object, ErrorDiagnostic>();
const restoreDiagnostics = new WeakMap<object, WorkspaceRestoreDiagnostic>();
-const diagnosticCapture = new AsyncLocalStorage<WeakMap<object, WorkspaceRestoreDiagnostic>>();
+const diagnosticCapture = new AsyncLocalStorage<{
+ restores: WeakMap<object, WorkspaceRestoreDiagnostic>;
+ wrapped: WeakMap<object, ErrorDiagnostic>;
+}>();
/** Keep each settlement's diagnostic receipt isolated across asynchronous work. */
export function withWorkspaceRestoreDiagnosticCapture<T>(operation: () => Promise<T>): Promise<T> {
- return diagnosticCapture.run(new WeakMap(), operation);
+ return diagnosticCapture.run({ restores: new WeakMap(), wrapped: new WeakMap() }, operation);
}
function readField(value: Record<string, unknown>, key: string): unknown {
@@ -108,7 +126,10 @@ function diagnostic(error: unknown): ErrorDiagnostic {
// SDKs wrap transport errors in a cause. Bound traversal, including cycles.
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
const value = current as Record<string, unknown>;
- const saved = wrappedDiagnostics.get(value);
+ const saved = (diagnosticCapture.getStore()?.wrapped ?? wrappedDiagnostics).get(value);
+ result.transferStep ??= saved?.transferStep;
+ result.transferFailureKind ??= saved?.transferFailureKind;
+ result.rpcCode ??= saved?.rpcCode;
const code = saved?.errorCode !== "unknown" && saved?.errorCode !== undefined
? saved.errorCode : readField(value, "code");
if (result.errorCode === "unknown" && typeof code === "string" && ERROR_CODES.has(code)) {
@@ -129,14 +150,27 @@ function diagnostic(error: unknown): ErrorDiagnostic {
}
/** Retain bounded fields across an existing error wrapper, without a raw cause. */
-export function preserveWorkspaceRestoreErrorDiagnostic<T extends object>(wrapper: T, source: unknown): T {
- wrappedDiagnostics.set(wrapper, diagnostic(source));
+export function preserveWorkspaceRestoreErrorDiagnostic<T extends object>(wrapper: T, source: unknown, transferEvidence?: unknown): T {
+ (diagnosticCapture.getStore()?.wrapped ?? wrappedDiagnostics).set(wrapper, { ...diagnostic(source), ...sanitizeTransferEvidence(transferEvidence) });
const scope = activeDiagnostic.getStore();
const failure = scope?.active ? scope.failures.get(source) : undefined;
if (failure) scope!.failures.set(wrapper, failure);
return wrapper;
}
+function sanitizeTransferEvidence(value: unknown): Pick<WorkspaceRestoreDiagnostic, "transferStep" | "transferFailureKind" | "rpcCode"> {
+ if (!value || typeof value !== "object") return {};
+ const record = value as Record<string, unknown>;
+ const step = readField(record, "transferStep");
+ const kind = readField(record, "transferFailureKind");
+ const rpcCode = readField(record, "rpcCode");
+ return {
+ ...(typeof step === "string" && TRANSFER_STEPS.has(step as TransferStep) ? { transferStep: step as TransferStep } : {}),
+ ...(typeof kind === "string" && TRANSFER_FAILURE_KINDS.has(kind as TransferFailureKind) ? { transferFailureKind: kind as TransferFailureKind } : {}),
+ ...(typeof rpcCode === "number" && RPC_CODES.has(rpcCode) ? { rpcCode } : {}),
+ };
+}
+
/** Decode persisted adapter metadata; never copy arbitrary properties or text. */
export function sanitizeWorkspaceRestoreDiagnostic(value: unknown): WorkspaceRestoreDiagnostic | undefined {
if (!value || typeof value !== "object") return undefined;
@@ -153,6 +187,7 @@ export function sanitizeWorkspaceRestoreDiagnostic(value: unknown): WorkspaceRes
&& typeof gitCommand === "string" && GIT_COMMANDS.has(gitCommand as WorkspaceRestoreGitCommand);
return {
phase,
+ ...(step === "workspace_transfer" || step === "asset_restore" ? sanitizeTransferEvidence(record) : {}),
...(typeof step === "string" && RESTORE_STEPS.has(step as WorkspaceRestoreStep)
? { step: step as WorkspaceRestoreStep } : {}),
errorCode: typeof code === "string" && ERROR_CODES.has(code) ? code : "unknown",
@@ -168,14 +203,14 @@ export function sanitizeWorkspaceRestoreDiagnostic(value: unknown): WorkspaceRes
export function getWorkspaceRestoreDiagnostic(error: unknown): WorkspaceRestoreDiagnostic | undefined {
return error && typeof error === "object"
- ? sanitizeWorkspaceRestoreDiagnostic((diagnosticCapture.getStore() ?? restoreDiagnostics).get(error)) : undefined;
+ ? sanitizeWorkspaceRestoreDiagnostic((diagnosticCapture.getStore()?.restores ?? restoreDiagnostics).get(error)) : undefined;
}
/** Preserve the scheduler-selected task's snapshot when errors share identity. */
export function recordWorkspaceRestoreDiagnostic(error: unknown, diagnostic: WorkspaceRestoreDiagnostic | undefined): void {
const safe = sanitizeWorkspaceRestoreDiagnostic(diagnostic);
if (error && typeof error === "object") {
- const receipts = diagnosticCapture.getStore() ?? restoreDiagnostics;
+ const receipts = diagnosticCapture.getStore()?.restores ?? restoreDiagnostics;
if (safe) receipts.set(error, safe);
else receipts.delete(error);
}
@@ -229,10 +264,10 @@ export async function withWorkspaceRestoreDiagnostics<T>(
} catch (error) {
const failure = scope.failures.get(error);
const step = failure?.step;
- const fields = { phase, ...(step ? { step } : {}), ...diagnostic(error),
+ const fields = sanitizeWorkspaceRestoreDiagnostic({ phase, ...(step ? { step } : {}), ...diagnostic(error),
...(phase === "workspace" && step === "git_integration" && failure?.gitCommand
? { gitCommand: failure.gitCommand, gitFailureKind: failure.gitFailureKind } : {}),
- };
+ })!;
recordWorkspaceRestoreDiagnostic(error, fields);
if (parent?.active && step) parent.failures.set(error, { ...failure, sequence: ++parent.sequence, step });
try {
diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync-diagnostics.test.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync-diagnostics.test.ts
new file mode 100644
index 0000000000..4e5e2da1ce
--- /dev/null
+++ b/packages/plugins/sandbox-providers/daytona/src/file-sync-diagnostics.test.ts
@@ -0,0 +1,113 @@
+import { promises as fs } from "node:fs";
+import { execFileSync } from "node:child_process";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, expect, it, vi } from "vitest";
+import { environmentSyncErrorData, withEnvironmentSyncErrorCapture, readEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk";
+vi.mock("./plugin.js", async () => {
+ const { NOOP_PLUGIN_TRACER } = await import("@paperclipai/plugin-sdk");
+ return { getPluginTracer: () => NOOP_PLUGIN_TRACER };
+});
+import { performSyncOut } from "./file-sync.js";
+
+const dirs: string[] = [];
+afterEach(async () => { await Promise.all(dirs.splice(0).map(dir => fs.rm(dir, { recursive: true, force: true }))); });
+
+it.each([
+ ["guard", "sandbox_guard", "command_failed", 42],
+ ["create", "archive_create", "command_failed", 2],
+ ["download", "archive_download", "download_failed", undefined],
+ ["missing", "archive_download", "download_missing", undefined],
+ ["file", "file_download", "download_failed", undefined],
+ ["file-missing", "file_download", "download_missing", undefined],
+] as const)("records producer evidence for %s without copying output or paths", async (failure, step, kind, exitCode) => {
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "daytona-diagnostic-"));
+ dirs.push(dir);
+ const sandbox = {
+ process: { executeCommand: async (command: string) => ({
+ exitCode: failure === "guard" ? 42 : failure === "create" && command.includes("tar -c") ? 2 : 0,
+ result: "private-command-output",
+ }) },
+ fs: {
+ downloadFiles: async (requests: Array<{ source: string }>) => failure.endsWith("missing") ? []
+ : requests.map(request => ({ source: request.source, error: "private-download-error" })),
+ deleteFile: async () => undefined,
+ },
+ };
+ await withEnvironmentSyncErrorCapture(async () => {
+ let caught: unknown;
+ try {
+ await performSyncOut({ sandbox: sandbox as never, remoteDir: "/workspace", timeoutSeconds: 30,
+ operations: [{ operationId: "fixture", files: [{
+ sourcePath: "/workspace/private-path", targetPath: path.join(dir, "target"), kind: failure.startsWith("file") ? "file" : "directory",
+ }] }],
+ });
+ } catch (error) { caught = error; }
+ expect(caught).toBeInstanceOf(Error);
+ const diagnostic = readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(caught) });
+ expect(diagnostic).toEqual({ errorCode: "unknown", transferStep: step, transferFailureKind: kind,
+ ...(exitCode !== undefined ? { exitCode } : {}),
+ });
+ expect(JSON.stringify(diagnostic)).not.toContain("private-");
+ expect(caught).not.toHaveProperty("transferStep");
+ expect(caught).not.toHaveProperty("exitCode");
+ });
+});
+
+it("keeps a thrown download exception identical and its nested safe code", async () => {
+ const original = Object.freeze(new Error("private-download-error", { cause: { code: "ECONNRESET", token: "private-token" } }));
+ const sandbox = {
+ process: { executeCommand: async () => ({ exitCode: 0 }) },
+ fs: { downloadFiles: async () => { throw original; }, deleteFile: async () => undefined },
+ };
+ await withEnvironmentSyncErrorCapture(async () => {
+ await expect(performSyncOut({ sandbox: sandbox as never, remoteDir: "/workspace", timeoutSeconds: 30,
+ operations: [{ operationId: "fixture", files: [{ sourcePath: "/workspace/private-path", targetPath: "/unused", kind: "directory" }] }],
+ })).rejects.toBe(original);
+ expect(readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(original) })).toEqual({
+ errorCode: "ECONNRESET", transferStep: "archive_download",
+ });
+ });
+});
+
+
+it.each(["file_finalize", "archive_validate", "archive_extract"] as const)("identifies the host-side %s failure without relaxing confinement", async (step) => {
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "daytona-host-diagnostic-"));
+ dirs.push(dir);
+ const target = path.join(dir, "target");
+ const archive = path.join(dir, "safe.tar");
+ if (step === "archive_extract") {
+ await fs.writeFile(path.join(dir, "safe.txt"), "fixture");
+ execFileSync("tar", ["-cf", archive, "-C", dir, "safe.txt"]);
+ await fs.writeFile(target, "preserve target");
+ }
+ const sandbox = {
+ process: { executeCommand: async () => ({ exitCode: 0 }) },
+ fs: {
+ downloadFiles: async (requests: Array<{ source: string; destination: string }>) => {
+ for (const request of requests) {
+ if (step === "archive_validate") await fs.writeFile(request.destination, "invalid private archive");
+ if (step === "archive_extract") await fs.copyFile(archive, request.destination);
+ }
+ return requests.map(request => ({ source: request.source }));
+ },
+ deleteFile: async () => undefined,
+ },
+ };
+ await withEnvironmentSyncErrorCapture(async () => {
+ let caught: unknown;
+ try {
+ await performSyncOut({ sandbox: sandbox as never, remoteDir: "/workspace", timeoutSeconds: 30,
+ operations: [{ operationId: "fixture", files: [{ sourcePath: "/workspace/private-path", targetPath: target,
+ kind: step === "file_finalize" ? "file" : "directory" }] }],
+ });
+ } catch (error) { caught = error; }
+ expect(caught).toBeInstanceOf(Error);
+ const diagnostic = readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(caught) });
+ expect(diagnostic?.transferStep).toBe(step);
+ expect(JSON.stringify(diagnostic)).not.toContain("private");
+ if (step === "file_finalize") expect(diagnostic?.errorCode).toBe("ENOENT");
+ if (step === "archive_validate") expect(diagnostic?.transferFailureKind).toBe("command_failed");
+ if (step === "archive_extract") expect(await fs.readFile(target, "utf8")).toBe("preserve target");
+ });
+});
diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts
index ad84d90d39..89847f9ddb 100644
--- a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts
@@ -14,6 +14,7 @@ import type {
PluginSyncFileMapping,
PluginSyncOperation,
} from "@paperclipai/plugin-sdk";
+import { recordEnvironmentSyncError, withEnvironmentSyncTransferStep } from "@paperclipai/plugin-sdk";
import { getPluginTracer } from "./plugin.js";
const execFileAsync = promisify(execFile);
@@ -318,9 +319,13 @@ export async function assertTarballEntriesConfined(
const validateLine = (line: Buffer) => {
// Count empty lines too, so whitespace cannot evade the parsing-work quota.
if (++entries > TAR_LISTING_MAX_ENTRIES) {
- throw new Error("Daytona syncOut tar listing entry limit exceeded (250000)");
+ throw recordEnvironmentSyncError(new Error("Daytona syncOut tar listing entry limit exceeded (250000)"), { transferFailureKind: "listing_entry_limit" });
+ }
+ try { assertTarListingLineConfined(line.toString("utf8")); }
+ catch (error) {
+ if (error instanceof UnsafeOutboundArchiveError) recordEnvironmentSyncError(error, { transferFailureKind: "unsafe_archive" });
+ throw error;
}
- assertTarListingLineConfined(line.toString("utf8"));
};
const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => {
child.once("error", (error) => { spawnError = error; });
@@ -331,11 +336,11 @@ export async function assertTarballEntriesConfined(
child.kill("SIGKILL");
};
const timer = setTimeout(() => {
- stop(new Error("Daytona syncOut tar listing validation timed out"));
+ stop(recordEnvironmentSyncError(new Error("Daytona syncOut tar listing validation timed out"), { transferFailureKind: "listing_timeout", errorCode: "ETIMEDOUT" }));
}, Math.max(1, Math.min(timeoutMs, TAR_LISTING_TIMEOUT_MS)));
child.stderr.on("data", (chunk: Buffer) => {
if (stderr.length + chunk.length > TAR_LISTING_MAX_STDERR_BYTES) {
- stop(new Error("Daytona syncOut tar listing diagnostics exceed the byte limit"));
+ stop(recordEnvironmentSyncError(new Error("Daytona syncOut tar listing diagnostics exceed the byte limit"), { transferFailureKind: "listing_stderr_limit" }));
return;
}
stderr = Buffer.concat([stderr, chunk]);
@@ -346,14 +351,14 @@ export async function assertTarballEntriesConfined(
const bytes = chunk as Buffer;
totalBytes += bytes.length;
if (totalBytes > TAR_LISTING_MAX_TOTAL_BYTES) {
- throw new Error("Daytona syncOut tar total listing byte limit exceeded (64 MiB)");
+ throw recordEnvironmentSyncError(new Error("Daytona syncOut tar total listing byte limit exceeded (64 MiB)"), { transferFailureKind: "listing_byte_limit" });
}
let start = 0;
while (start < bytes.length) {
const newline = bytes.indexOf(10, start);
const end = newline < 0 ? bytes.length : newline;
if (pending.length + end - start > TAR_LISTING_MAX_LINE_BYTES) {
- throw new Error("Daytona syncOut refusing tarball with an entry listing exceeding the byte limit");
+ throw recordEnvironmentSyncError(new Error("Daytona syncOut refusing tarball with an entry listing exceeding the byte limit"), { transferFailureKind: "listing_line_limit" });
}
pending = Buffer.concat([pending, bytes.subarray(start, end)]);
if (newline < 0) break;
@@ -367,7 +372,9 @@ export async function assertTarballEntriesConfined(
if (failure) throw failure;
if (spawnError) throw spawnError;
if (result.code !== 0) {
- throw new Error(`Daytona syncOut tar listing failed (${result.signal ?? result.code}): ${stderr.toString("utf8").trim()}`);
+ throw recordEnvironmentSyncError(new Error(`Daytona syncOut tar listing failed (${result.signal ?? result.code}): ${stderr.toString("utf8").trim()}`), {
+ transferFailureKind: "command_failed", exitCode: result.code ?? undefined,
+ });
}
} catch (error) {
stop(error instanceof Error ? error : new Error(String(error)));
@@ -381,11 +388,13 @@ export async function assertTarballEntriesConfined(
async function extractHostTarball(input: { archivePath: string; localDir: string }): Promise<void> {
// The archive is sandbox-authored and untrusted: validate every member (and
// link target) is confined before letting host-side tar write a single byte.
- await assertTarballEntriesConfined(input.archivePath);
- await fs.mkdir(input.localDir, { recursive: true });
- await execFileAsync("tar", ["-xf", input.archivePath, "-C", input.localDir], {
- env: { ...process.env, COPYFILE_DISABLE: "1" },
- maxBuffer: 32 * 1024 * 1024,
+ await withEnvironmentSyncTransferStep("archive_validate", () => assertTarballEntriesConfined(input.archivePath));
+ await withEnvironmentSyncTransferStep("archive_extract", async () => {
+ await fs.mkdir(input.localDir, { recursive: true });
+ await execFileAsync("tar", ["-xf", input.archivePath, "-C", input.localDir], {
+ env: { ...process.env, COPYFILE_DISABLE: "1" },
+ maxBuffer: 32 * 1024 * 1024,
+ });
});
}
@@ -423,7 +432,9 @@ async function assertSandboxCommandOkWithOutput(
const result = await sandbox.process.executeCommand(command, undefined, undefined, timeoutSeconds);
if ((result.exitCode ?? 1) !== 0) {
const detail = (result.result ?? result.artifacts?.stdout ?? "").toString().trim();
- throw new Error(`Daytona ${label} command failed (exit ${result.exitCode ?? "unknown"})${detail ? `: ${detail}` : ""}`);
+ throw recordEnvironmentSyncError(new Error(`Daytona ${label} command failed (exit ${result.exitCode ?? "unknown"})${detail ? `: ${detail}` : ""}`), {
+ transferFailureKind: "command_failed", exitCode: result.exitCode,
+ });
}
return (result.result ?? result.artifacts?.stdout ?? "").toString();
}
@@ -1143,17 +1154,14 @@ async function syncOutFileMappings(input: {
const { sandbox, mappings, remoteDir, timeoutSeconds } = input;
if (mappings.length === 0) return { filesTransferred: 0, bytesTransferred: 0 };
- for (const mapping of mappings) {
- assertConfinedSandboxPath(remoteDir, mapping.sourcePath, "source");
- }
// Close the validation→download TOCTOU: instead of re-opening each mutable
// source, validate-and-snapshot it in one atomic sandbox-side step and download
// the immutable snapshot. `snapshots` is index-aligned with `mappings`.
- const snapshots = await snapshotOutboundFileSources({
- sandbox,
- remoteDir,
- sources: mappings.map((mapping) => mapping.sourcePath),
- timeoutSeconds,
+ const snapshots = await withEnvironmentSyncTransferStep("sandbox_guard", async () => {
+ for (const mapping of mappings) assertConfinedSandboxPath(remoteDir, mapping.sourcePath, "source");
+ return snapshotOutboundFileSources({
+ sandbox, remoteDir, sources: mappings.map((mapping) => mapping.sourcePath), timeoutSeconds,
+ });
});
const requests: FileDownloadRequest[] = [];
@@ -1179,6 +1187,7 @@ async function syncOutFileMappings(input: {
await fs.mkdir(path.dirname(entry.target), { recursive: true });
}
} catch (error) {
+ recordEnvironmentSyncError(error, { transferStep: "file_finalize" });
await cleanup();
throw error;
}
@@ -1193,7 +1202,7 @@ async function syncOutFileMappings(input: {
try {
// One batched bulk download for all file mappings, reading the snapshots.
// `transfer` span: the real byte download — `sandbox.fs.downloadFiles`.
- responses = await withProviderSpan({
+ responses = await withEnvironmentSyncTransferStep("file_download", () => withProviderSpan({
name: "transfer",
wallMsAttr: SPAN_ATTR.transferWallMs,
attributes: {
@@ -1201,7 +1210,7 @@ async function syncOutFileMappings(input: {
[SPAN_ATTR.transferDirection]: "outbound",
},
run: () => sandbox.fs.downloadFiles(requests, timeoutSeconds),
- });
+ }));
} catch (error) {
await cleanup();
throw error;
@@ -1215,9 +1224,9 @@ async function syncOutFileMappings(input: {
const response = bySource.get(entry.snapshot);
if (!response || response.error) {
await cleanup();
- throw new Error(
+ throw recordEnvironmentSyncError(new Error(
`Daytona syncOut download failed for ${entry.source}: ${response?.error ?? "no response returned"}`,
- );
+ ), { transferStep: "file_download", transferFailureKind: response ? "download_failed" : "download_missing" });
}
}
@@ -1233,6 +1242,7 @@ async function syncOutFileMappings(input: {
await fs.rename(entry.temp, entry.target);
}
} catch (error) {
+ recordEnvironmentSyncError(error, { transferStep: "file_finalize" });
await cleanup();
throw error;
}
@@ -1251,16 +1261,15 @@ async function syncOutDirectoryMapping(input: {
onArchiveRecovery?: () => void;
}): Promise<{ filesTransferred: number; bytesTransferred: number }> {
const { sandbox, mapping, remoteDir, timeoutSeconds } = input;
- assertConfinedSandboxPath(remoteDir, mapping.sourcePath, "source");
// Count the serial sandbox round trips before the transfer, so the transfer
// span records how much of the wall time is guard cost.
let guardRoundTrips = 0;
- await assertSandboxPathsConfined({
- sandbox,
- remoteDir,
- paths: [mapping.sourcePath],
- timeoutSeconds,
- label: "outbound symlink-escape guard",
+ await withEnvironmentSyncTransferStep("sandbox_guard", async () => {
+ assertConfinedSandboxPath(remoteDir, mapping.sourcePath, "source");
+ await assertSandboxPathsConfined({
+ sandbox, remoteDir, paths: [mapping.sourcePath], timeoutSeconds,
+ label: "outbound symlink-escape guard",
+ });
});
guardRoundTrips += 1;
@@ -1310,10 +1319,11 @@ async function syncOutDirectoryMapping(input: {
let bytesTransferred = 0;
try {
for (let attempt = 0; attempt < 2; attempt += 1) {
- await assertSandboxCommandOk(sandbox, `sh -c ${shellQuote(attempt === 0 ? tarScript : confinedEntriesScript)}`, timeoutSeconds, "syncOut tar");
+ await withEnvironmentSyncTransferStep("archive_create", () =>
+ assertSandboxCommandOk(sandbox, `sh -c ${shellQuote(attempt === 0 ? tarScript : confinedEntriesScript)}`, timeoutSeconds, "syncOut tar"));
guardRoundTrips += 1;
// `transfer` span: the real byte download — `sandbox.fs.downloadFiles`.
- const responses = await withProviderSpan({
+ const responses = await withEnvironmentSyncTransferStep("archive_download", () => withProviderSpan({
name: "transfer",
wallMsAttr: SPAN_ATTR.transferWallMs,
attributes: {
@@ -1322,14 +1332,14 @@ async function syncOutDirectoryMapping(input: {
},
run: () =>
sandbox.fs.downloadFiles([{ source: remoteTar, destination: localTar }], timeoutSeconds),
- });
+ }));
const response = responses.find((entry) => entry.source === remoteTar) ?? responses[0];
if (!response || response.error) {
- throw new Error(
+ throw recordEnvironmentSyncError(new Error(
`Daytona syncOut directory download failed for ${mapping.sourcePath}: ${response?.error ?? "no response returned"}`,
- );
+ ), { transferStep: "archive_download", transferFailureKind: response ? "download_failed" : "download_missing" });
}
- bytesTransferred += (await fs.stat(localTar)).size;
+ bytesTransferred += await withEnvironmentSyncTransferStep("archive_download", async () => (await fs.stat(localTar)).size);
try {
await extractHostTarball({ archivePath: localTar, localDir: mapping.targetPath });
break;
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
index 6022d9b3fe..79eb85bd0a 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
@@ -36,7 +36,7 @@ import plugin, {
__setDaytonaPluginContextForTest,
} from "./plugin.js";
import type { PluginContext } from "@paperclipai/plugin-sdk";
-import { environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
+import { environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
import manifest from "./manifest.js";
import { parseTarVerboseListingLine, splitLinkEntryOnce } from "./file-sync.js";
@@ -5263,33 +5263,32 @@ describe("daytona native file-sync hooks", () => {
});
});
- it("classifies a deleted sandbox during syncOut with a stable unrecoverable code", async () => {
+ it.each(["sandbox_access", "file_download"] as const)("keeps the deleted-sandbox policy and bounded %s evidence", async (step) => {
const hostDir = await makeHostDir();
- mockGet.mockRejectedValue(
- new MockDaytonaNotFoundError("provider detail must not escape"),
- );
-
- await expect(
- plugin.definition.onEnvironmentSyncOut?.({
- driverKey: "daytona",
- companyId: "company-1",
- environmentId: "env-1",
- config: { timeoutMs: 300000, reuseLease: true },
- lease: syncLease(),
- operations: [
- {
- operationId: "sync-op-missing-sandbox",
- files: [
- {
- sourcePath: `${REMOTE_DIR}/out/result.txt`,
- targetPath: path.join(hostDir, "result.txt"),
- kind: "file",
- },
- ],
- },
- ],
- }),
- ).rejects.toThrow("daytona_sandbox_not_found");
+ const source = Object.freeze(Object.assign(new MockDaytonaNotFoundError("private-provider-detail"), {
+ status: 404, cause: { code: "EIO", token: "private-provider-token" },
+ }));
+ if (step === "sandbox_access") mockGet.mockRejectedValue(source);
+ else {
+ const sandbox = createMockSandbox();
+ sandbox.fs.downloadFiles.mockRejectedValue(source);
+ mockGet.mockResolvedValue(sandbox);
+ }
+ await withEnvironmentSyncErrorCapture(async () => {
+ const error = await plugin.definition.onEnvironmentSyncOut!({
+ driverKey: "daytona", companyId: "company-1", environmentId: "env-1",
+ config: { timeoutMs: 300000, reuseLease: true }, lease: syncLease(),
+ operations: [{ operationId: "sync-op-missing-sandbox", files: [{
+ sourcePath: `${REMOTE_DIR}/out/result.txt`, targetPath: path.join(hostDir, "result.txt"), kind: "file",
+ }] }],
+ }).catch(error => error);
+ expect(error).toBeInstanceOf(Error);
+ expect(error.message).toBe("daytona_sandbox_not_found");
+ expect(error).not.toHaveProperty("cause");
+ const diagnostic = readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(error) });
+ expect(diagnostic).toEqual({ errorCode: "EIO", httpStatus: 404, transferStep: step });
+ expect(JSON.stringify(diagnostic)).not.toContain("private-provider-");
+ });
});
it("syncOut snapshot guard re-checks the resolved source is a non-symlink regular file immediately before copying (validation→copy TOCTOU)", async () => {
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
index 9c085d6a69..bbfa70a049 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
@@ -9,7 +9,7 @@ import type {
Resources,
Sandbox,
} from "@daytonaio/sdk";
-import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentCreationCleanupError, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
+import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentCreationCleanupError, readEnvironmentCreationCleanupError, withEnvironmentSyncTransferStep, preserveEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk";
import type {
PluginContext,
PluginEnvironmentCreationCleanup,
@@ -3271,8 +3271,11 @@ const plugin = definePlugin({
};
try {
return await withSandboxActivityGate(scope, async () => {
- const sandbox = await getSandbox(scope, { bypassTeardownGate: true });
- await ensureSandboxStarted(sandbox, timeoutSeconds);
+ const sandbox = await withEnvironmentSyncTransferStep("sandbox_access", async () => {
+ const resolved = await getSandbox(scope, { bypassTeardownGate: true });
+ await ensureSandboxStarted(resolved, timeoutSeconds);
+ return resolved;
+ });
const result = await performSyncOut({
sandbox,
operations: params.operations,
@@ -3288,7 +3291,7 @@ const plugin = definePlugin({
// unexported workspace bytes no longer exist. Convert the SDK class to a
// stable cross-worker message; every other error remains retryable.
if (error instanceof DaytonaNotFoundError) {
- throw new Error("daytona_sandbox_not_found");
+ throw preserveEnvironmentSyncErrorDiagnostic(new Error("daytona_sandbox_not_found"), error);
}
throw error;
}
diff --git a/packages/plugins/sandbox-providers/daytona/src/tar-listing.test.ts b/packages/plugins/sandbox-providers/daytona/src/tar-listing.test.ts
index 796e93bc00..cf0fb98a77 100644
--- a/packages/plugins/sandbox-providers/daytona/src/tar-listing.test.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/tar-listing.test.ts
@@ -4,6 +4,7 @@ import path from "node:path";
import { afterEach, expect, it, vi } from "vitest";
vi.mock("./plugin.js", () => ({ getPluginTracer: () => undefined }));
+import { environmentSyncErrorData, withEnvironmentSyncErrorCapture, withEnvironmentSyncTransferStep, readEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk";
import { assertTarballEntriesConfined } from "./file-sync.js";
const temporaryDirectories: string[] = [];
@@ -100,18 +101,28 @@ it.each([false, true])("bounds aggregate listing bytes at 64 MiB (over quota: %s
const line = "-rw-r--r-- 0/0 0 2026-09-27 12:00 ".padEnd(4095, "x") + "\n";
const count = 64 * 1024 * 1024 / Buffer.byteLength(line) + Number(overQuota);
const checkReaped = await fakeTar(listingWriter(line, count));
- const result = assertTarballEntriesConfined("unused.tar");
- if (overQuota) await expect(result).rejects.toThrow("total listing byte limit");
- else await expect(result).resolves.toBeUndefined();
+ await withEnvironmentSyncErrorCapture(async () => {
+ const result = assertTarballEntriesConfined("unused.tar");
+ if (overQuota) {
+ const error = await result.catch(error => error);
+ expect(error.message).toContain("total listing byte limit");
+ expect(readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(error) })?.transferFailureKind).toBe("listing_byte_limit");
+ } else await expect(result).resolves.toBeUndefined();
+ });
await checkReaped();
}, 30_000);
it.each([false, true])("bounds aggregate listing entries at 250,000 (over quota: %s)", async (overQuota) => {
// The over-quota member has no newline: EOF must pass the same admission gate.
const checkReaped = await fakeTar(listingWriter(safeLine + "\n", 250_000, overQuota ? safeLine : ""));
- const result = assertTarballEntriesConfined("unused.tar");
- if (overQuota) await expect(result).rejects.toThrow("listing entry limit");
- else await expect(result).resolves.toBeUndefined();
+ await withEnvironmentSyncErrorCapture(async () => {
+ const result = assertTarballEntriesConfined("unused.tar");
+ if (overQuota) {
+ const error = await result.catch(error => error);
+ expect(error.message).toContain("listing entry limit");
+ expect(readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(error) })?.transferFailureKind).toBe("listing_entry_limit");
+ } else await expect(result).resolves.toBeUndefined();
+ });
await checkReaped();
}, 30_000);
@@ -189,3 +200,26 @@ it("rejects spawn failures without hanging", async () => {
vi.stubEnv("PATH", await temporaryDirectory());
await expect(assertTarballEntriesConfined("unused.tar")).rejects.toThrow("ENOENT");
});
+
+
+it.each([
+ ["process.exitCode = 2;", "command_failed", 2, "unknown"],
+ ['process.stdout.write("x".repeat(65_537)); setInterval(() => {}, 1_000);', "listing_line_limit", undefined, "unknown"],
+ ['process.stderr.write("x".repeat(65_537)); setInterval(() => {}, 1_000);', "listing_stderr_limit", undefined, "unknown"],
+ ['process.stdout.write("invalid-private-listing");', "unsafe_archive", undefined, "unknown"],
+ ['setInterval(() => {}, 1_000);', "listing_timeout", undefined, "ETIMEDOUT"],
+] as const)("records bounded listing evidence and reaps the child (%s)", async (script, kind, exitCode, errorCode) => {
+ const checkReaped = await fakeTar(script);
+ await withEnvironmentSyncErrorCapture(async () => {
+ let caught: unknown;
+ try { await withEnvironmentSyncTransferStep("archive_validate", () => assertTarballEntriesConfined("private-archive.tar", 2_000)); }
+ catch (error) { caught = error; }
+ expect(caught).toBeInstanceOf(Error);
+ const diagnostic = readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(caught) });
+ expect(diagnostic).toEqual({ errorCode, transferStep: "archive_validate", transferFailureKind: kind,
+ ...(exitCode !== undefined ? { exitCode } : {}),
+ });
+ expect(JSON.stringify(diagnostic)).not.toContain("private-");
+ });
+ await checkReaped();
+});
diff --git a/packages/plugins/sdk/src/environment-sync-error.ts b/packages/plugins/sdk/src/environment-sync-error.ts
index 0450310a07..a630bb126e 100644
--- a/packages/plugins/sdk/src/environment-sync-error.ts
+++ b/packages/plugins/sdk/src/environment-sync-error.ts
@@ -1,8 +1,67 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
/** Diagnostic evidence only. Never use this envelope to authorize cleanup or retry. */
export interface PluginEnvironmentSyncErrorDiagnostic {
errorCode: string;
httpStatus?: number;
exitCode?: number;
+ transferStep?: EnvironmentSyncTransferStep;
+ transferFailureKind?: EnvironmentSyncTransferFailureKind;
+}
+
+const TRANSFER_STEPS = new Set([
+ "sandbox_access", "sandbox_guard", "file_download", "file_finalize",
+ "archive_create", "archive_download", "archive_validate", "archive_extract",
+] as const);
+export type EnvironmentSyncTransferStep = typeof TRANSFER_STEPS extends Set<infer T> ? T : never;
+const TRANSFER_FAILURE_KINDS = new Set([
+ "command_failed", "download_failed", "download_missing", "unsafe_archive",
+ "listing_timeout", "listing_entry_limit", "listing_byte_limit", "listing_line_limit", "listing_stderr_limit",
+] as const);
+export type EnvironmentSyncTransferFailureKind = typeof TRANSFER_FAILURE_KINDS extends Set<infer T> ? T : never;
+interface Capture {
+ active: boolean;
+ sequence: number;
+ errors: WeakMap<object, { sequence: number; diagnostic: PluginEnvironmentSyncErrorDiagnostic }>;
+}
+const capture = new AsyncLocalStorage<Capture>();
+
+/** Scope evidence to one outbound RPC, including its error serialization. */
+export async function withEnvironmentSyncErrorCapture<T>(operation: () => Promise<T>): Promise<T> {
+ const scope: Capture = { active: true, sequence: 0, errors: new WeakMap() };
+ return capture.run(scope, async () => {
+ try { return await operation(); }
+ finally { scope.active = false; }
+ });
+}
+
+/** Producer-owned evidence only; never mutate, wrap, or retain a raw cause. */
+export function recordEnvironmentSyncError<T>(error: T, diagnostic: Partial<PluginEnvironmentSyncErrorDiagnostic>): T {
+ const scope = capture.getStore();
+ const safe = sanitize(diagnostic);
+ if (scope?.active && error && typeof error === "object") {
+ if (safe) scope.errors.set(error, { sequence: ++scope.sequence, diagnostic: safe });
+ else scope.errors.delete(error);
+ }
+ return error;
+}
+
+/** A nested failing stage wins; a later attempt gets fresh attribution. */
+export async function withEnvironmentSyncTransferStep<T>(step: EnvironmentSyncTransferStep, operation: () => Promise<T>): Promise<T> {
+ const scope = capture.getStore();
+ if (!scope?.active || !TRANSFER_STEPS.has(step)) return await operation();
+ const started = scope.sequence;
+ try { return await operation(); }
+ catch (error) {
+ if (scope.active && error && typeof error === "object") {
+ const saved = scope.errors.get(error);
+ const diagnostic = saved && saved.sequence > started ? saved.diagnostic : { errorCode: "unknown" };
+ scope.errors.set(error, {
+ sequence: ++scope.sequence, diagnostic: { ...diagnostic, transferStep: diagnostic.transferStep ?? step },
+ });
+ }
+ throw error;
+ }
}
const SCHEMA = "paperclip/environment-sync-error/v1";
@@ -29,13 +88,23 @@ function sanitize(value: unknown): PluginEnvironmentSyncErrorDiagnostic | undefi
const errorCode = typeof code === "string" && ERROR_CODES.has(code) ? code : "unknown";
const httpStatus = integer(field(value, "httpStatus"), 400, 599);
const exitCode = integer(field(value, "exitCode"), 1, 255);
- if (errorCode === "unknown" && httpStatus === undefined && exitCode === undefined) return undefined;
- return { errorCode, ...(httpStatus !== undefined ? { httpStatus } : {}), ...(exitCode !== undefined ? { exitCode } : {}) };
+ const step = field(value, "transferStep");
+ const kind = field(value, "transferFailureKind");
+ const transferStep = typeof step === "string" && TRANSFER_STEPS.has(step as EnvironmentSyncTransferStep) ? step as EnvironmentSyncTransferStep : undefined;
+ const transferFailureKind = typeof kind === "string" && TRANSFER_FAILURE_KINDS.has(kind as EnvironmentSyncTransferFailureKind) ? kind as EnvironmentSyncTransferFailureKind : undefined;
+ if (errorCode === "unknown" && httpStatus === undefined && exitCode === undefined && !transferStep && !transferFailureKind) return undefined;
+ return { errorCode, ...(httpStatus !== undefined ? { httpStatus } : {}), ...(exitCode !== undefined ? { exitCode } : {}),
+ ...(transferStep ? { transferStep } : {}), ...(transferFailureKind ? { transferFailureKind } : {}),
+ };
}
/** Capture known codes and numbers, never arbitrary error data, text, or names. */
-export function environmentSyncErrorData(error: unknown): unknown {
+function captureDiagnostic(error: unknown): PluginEnvironmentSyncErrorDiagnostic | undefined {
const diagnostic: PluginEnvironmentSyncErrorDiagnostic = { errorCode: "unknown" };
+ const scope = capture.getStore();
+ if (scope?.active && error && typeof error === "object") {
+ Object.assign(diagnostic, scope.errors.get(error)?.diagnostic);
+ }
let current = error;
// Bound nested SDK causes, including cyclic errors and throwing getters.
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
@@ -47,8 +116,18 @@ export function environmentSyncErrorData(error: unknown): unknown {
diagnostic.exitCode ??= integer(field(current, "exitCode"), 1, 255) ?? integer(code, 1, 255);
current = field(current, "cause");
}
- const safe = sanitize(diagnostic);
- return safe ? { schema: SCHEMA, diagnostic: safe } : undefined;
+ return sanitize(diagnostic);
+}
+
+export function environmentSyncErrorData(error: unknown): unknown {
+ const diagnostic = captureDiagnostic(error);
+ return diagnostic ? { schema: SCHEMA, diagnostic } : undefined;
+}
+
+/** Retain bounded evidence across an existing wrapper, without adding a raw cause. */
+export function preserveEnvironmentSyncErrorDiagnostic<T>(wrapper: T, source: unknown): T {
+ const diagnostic = captureDiagnostic(source);
+ return recordEnvironmentSyncError(wrapper, diagnostic ?? { errorCode: "unknown" });
}
/** Revalidate the worker envelope before adapting it into host-only diagnostics. */
diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts
index 7610aaa9a1..5043091251 100644
--- a/packages/plugins/sdk/src/index.ts
+++ b/packages/plugins/sdk/src/index.ts
@@ -452,7 +452,7 @@ export {
export { PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError } from "./environment-creation-cleanup.js";
export type { PluginEnvironmentCreationCleanup } from "./environment-creation-cleanup.js";
-export { readEnvironmentSyncErrorDiagnostic } from "./environment-sync-error.js";
+export { preserveEnvironmentSyncErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, withEnvironmentSyncTransferStep, recordEnvironmentSyncError } from "./environment-sync-error.js";
export type { PluginEnvironmentSyncErrorDiagnostic } from "./environment-sync-error.js";
export type { AiConnectionPool, AiConnectionPoolConfig, AiConnectionPoolMember, AiConnectionRouterRequest, AiConnectionRouterResult, AiConnectionRouterSelection } from "@paperclipai/shared";
diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts
index d6781c1b6d..a374e31922 100644
--- a/packages/plugins/sdk/src/worker-rpc-host.ts
+++ b/packages/plugins/sdk/src/worker-rpc-host.ts
@@ -1,6 +1,6 @@
import type { AiConnectionRouterRequest, AiConnectionRouterResult } from "@paperclipai/shared";
import { environmentCreationCleanupErrorData } from "./environment-creation-cleanup.js";
-import { environmentSyncErrorData } from "./environment-sync-error.js";
+import { environmentSyncErrorData, withEnvironmentSyncErrorCapture } from "./environment-sync-error.js";
import { createPluginIdleDrain } from "./idle-drain.js";
/**
* Worker-side RPC host — runs inside the child process spawned by the host.
@@ -1570,6 +1570,13 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost
* Dispatches to the correct handler based on the method name.
*/
async function handleHostRequest(request: JsonRpcRequest): Promise<void> {
+ if (request.method === "environmentSyncOut") {
+ return withEnvironmentSyncErrorCapture(() => handleHostRequestInScope(request));
+ }
+ return handleHostRequestInScope(request);
+ }
+
+ async function handleHostRequestInScope(request: JsonRpcRequest): Promise<void> {
const { id, method, params } = request;
let done: (() => void) | undefined;
try {
diff --git a/packages/plugins/sdk/tests/environment-sync-error.test.ts b/packages/plugins/sdk/tests/environment-sync-error.test.ts
index 26d9805493..c2480a707c 100644
--- a/packages/plugins/sdk/tests/environment-sync-error.test.ts
+++ b/packages/plugins/sdk/tests/environment-sync-error.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic } from "../src/environment-sync-error.js";
+import { preserveEnvironmentSyncErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, withEnvironmentSyncTransferStep, recordEnvironmentSyncError } from "../src/environment-sync-error.js";
const schema = "paperclip/environment-sync-error/v1";
@@ -46,3 +46,70 @@ describe("environment sync diagnostic envelope", () => {
expect(readEnvironmentSyncErrorDiagnostic(Object.defineProperty({}, "data", { get() { throw new Error("private"); } }))).toBeUndefined();
});
});
+
+
+describe("request-local transfer evidence", () => {
+ it("preserves frozen error identity, nested attribution, and later-attempt attribution", async () => {
+ const error = Object.freeze(new Error("private-message"));
+ const before = Object.getOwnPropertyDescriptors(error);
+ await withEnvironmentSyncErrorCapture(async () => {
+ await expect(withEnvironmentSyncTransferStep("archive_download", () =>
+ withEnvironmentSyncTransferStep("sandbox_access", async () => { throw error; }))).rejects.toBe(error);
+ expect(readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(error) })?.transferStep).toBe("sandbox_access");
+ await expect(withEnvironmentSyncTransferStep("archive_create", async () => { throw error; })).rejects.toBe(error);
+ expect(readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(error) })?.transferStep).toBe("archive_create");
+ });
+ expect(environmentSyncErrorData(error)).toBeUndefined();
+ expect(Object.getOwnPropertyDescriptors(error)).toEqual(before);
+ await withEnvironmentSyncErrorCapture(async () => expect(environmentSyncErrorData(error)).toBeUndefined());
+ });
+
+ it("does not trust labels on raw errors, their causes, or unknown producer values", async () => {
+ const error = { transferStep: "archive_create", transferFailureKind: "command_failed",
+ cause: { transferStep: "archive_validate", message: "private-message" } };
+ await withEnvironmentSyncErrorCapture(async () => {
+ expect(environmentSyncErrorData(error)).toBeUndefined();
+ recordEnvironmentSyncError(error, { transferStep: "private-stage", transferFailureKind: "private-kind", exitCode: 900 } as never);
+ expect(environmentSyncErrorData(error)).toBeUndefined();
+ });
+ expect(readEnvironmentSyncErrorDiagnostic({ data: { schema, diagnostic: {
+ errorCode: "unknown", transferStep: "private-stage", transferFailureKind: "private-kind", rpcCode: -32003,
+ } } })).toBeUndefined();
+ });
+
+ it("omits evidence from detached work after the RPC scope closes", async () => {
+ const error = new Error("private-message");
+ let resume!: () => void;
+ const gate = new Promise<void>(resolve => { resume = resolve; });
+ let detached!: Promise<unknown>;
+ await withEnvironmentSyncErrorCapture(async () => {
+ detached = withEnvironmentSyncTransferStep("archive_create", async () => { await gate; throw error; })
+ .catch(failure => environmentSyncErrorData(failure));
+ });
+ resume();
+ expect(await detached).toBeUndefined();
+ });
+});
+
+
+it("copies safe evidence into an existing wrapper only within its request", async () => {
+ const source = Object.freeze(Object.assign(new Error("private-message"), { status: 404, cause: { code: "EIO", token: "private-token" } }));
+ const wrapper = Object.freeze(new Error("stable-policy-message"));
+ await withEnvironmentSyncErrorCapture(async () => {
+ await expect(withEnvironmentSyncTransferStep("file_download", async () => { throw source; })).rejects.toBe(source);
+ expect(preserveEnvironmentSyncErrorDiagnostic(wrapper, source)).toBe(wrapper);
+ expect(readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(wrapper) })).toEqual({
+ errorCode: "EIO", httpStatus: 404, transferStep: "file_download",
+ });
+ expect(wrapper).not.toHaveProperty("cause");
+ expect(wrapper.message).toBe("stable-policy-message");
+ // Reusing a wrapper for a later untyped failure must clear the earlier evidence.
+ preserveEnvironmentSyncErrorDiagnostic(wrapper, new Error("private-untyped-message"));
+ expect(environmentSyncErrorData(wrapper)).toBeUndefined();
+ });
+ expect(environmentSyncErrorData(wrapper)).toBeUndefined();
+ await withEnvironmentSyncErrorCapture(async () => {
+ preserveEnvironmentSyncErrorDiagnostic(wrapper, source);
+ expect(readEnvironmentSyncErrorDiagnostic({ data: environmentSyncErrorData(wrapper) })).toEqual({ errorCode: "EIO", httpStatus: 404 });
+ });
+});
diff --git a/server/src/__tests__/environment-sync-out-error.test.ts b/server/src/__tests__/environment-sync-out-error.test.ts
index 36407d2453..8ca1a0b7c7 100644
--- a/server/src/__tests__/environment-sync-out-error.test.ts
+++ b/server/src/__tests__/environment-sync-out-error.test.ts
@@ -3,6 +3,7 @@ import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import {
createRequest, definePlugin, isJsonRpcResponse, JsonRpcCallError, parseMessage,
+ readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncTransferStep, recordEnvironmentSyncError,
PLUGIN_RPC_ERROR_CODES, serializeMessage, startWorkerRpcHost, type JsonRpcResponse,
} from "@paperclipai/plugin-sdk";
import {
@@ -53,8 +54,9 @@ describe("environment sync-out restore diagnostics", () => {
await expect(withWorkspaceRestoreDiagnostics("workspace", () => withWorkspaceRestoreStep("workspace_transfer", async () => {
throw preserveEnvironmentSyncOutErrorDiagnostic(error);
}), async (line) => { logs.push(line); })).rejects.toBe(error);
- expect(getWorkspaceRestoreDiagnostic(error)).toEqual({ phase: "workspace", step: "workspace_transfer", errorCode: "ETIMEDOUT", httpStatus: 504 });
- expect(logs).toEqual(['[paperclip] Workspace restore diagnostic: {"phase":"workspace","step":"workspace_transfer","errorCode":"ETIMEDOUT","httpStatus":504}\n']);
+ expect(getWorkspaceRestoreDiagnostic(error)).toEqual({ phase: "workspace", step: "workspace_transfer", errorCode: "ETIMEDOUT", httpStatus: 504, rpcCode: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR });
+ expect(logs).toHaveLength(1);
+ expect(JSON.parse(logs[0].slice(logs[0].indexOf("{")))).toEqual(getWorkspaceRestoreDiagnostic(error));
expect({ message: error.message, code: error.code, data: error.data }).toEqual(before);
expect(error).not.toHaveProperty("cause");
});
@@ -76,3 +78,69 @@ describe("environment sync-out restore diagnostics", () => {
expect(preserveEnvironmentSyncOutErrorDiagnostic(null)).toBeNull();
});
});
+
+
+it("isolates overlapping and sequential worker calls that throw the same object", async () => {
+ const original = Object.freeze(new Error("Transfer failed"));
+ const before = Object.getOwnPropertyDescriptors(original);
+ const stdin = new PassThrough();
+ const stdout = new PassThrough();
+ const lines = createInterface({ input: stdout });
+ let release!: () => void;
+ const gate = new Promise<void>(resolve => { release = resolve; });
+ let started!: () => void;
+ const ready = new Promise<void>(resolve => { started = resolve; });
+ const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({
+ async setup() {},
+ async onEnvironmentSyncOut(params) {
+ if (params.operations?.[0]?.operationId === "first") {
+ return withEnvironmentSyncTransferStep("archive_create", async () => {
+ recordEnvironmentSyncError(original, { exitCode: 2, transferFailureKind: "command_failed" });
+ started(); await gate; throw original;
+ });
+ }
+ if (params.operations?.[0]?.operationId === "second") {
+ return withEnvironmentSyncTransferStep("file_download", async () => { throw original; });
+ }
+ throw original;
+ },
+ async onEnvironmentSyncIn() { throw original; },
+ }) });
+ const call = (id: string, method = "environmentSyncOut") => new Promise<JsonRpcCallError>((resolve) => {
+ const listener = (wire: string) => {
+ const message = parseMessage(wire);
+ if (isJsonRpcResponse(message) && message.id === id && message.error) {
+ lines.off("line", listener); resolve(new JsonRpcCallError(message.error));
+ }
+ };
+ lines.on("line", listener);
+ stdin.write(serializeMessage(createRequest(method, { operations: [{ operationId: id, files: [] }] }, id)));
+ });
+ try {
+ const first = call("first");
+ await ready;
+ const second = await call("second");
+ expect(readEnvironmentSyncErrorDiagnostic(second)).toEqual({ errorCode: "unknown", transferStep: "file_download" });
+ release();
+ expect(readEnvironmentSyncErrorDiagnostic(await first)).toEqual({
+ errorCode: "unknown", transferStep: "archive_create", transferFailureKind: "command_failed", exitCode: 2,
+ });
+ expect((await call("third")).data).toBeUndefined();
+ expect((await call("unrelated", "environmentSyncIn")).data).toBeUndefined();
+ expect(Object.getOwnPropertyDescriptors(original)).toEqual(before);
+ } finally { release(); worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); }
+});
+
+it.each([
+ [new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.TIMEOUT, message: "private-rpc-message" }), PLUGIN_RPC_ERROR_CODES.TIMEOUT],
+ [new JsonRpcCallError({ code: 123, message: "private-rpc-message" }), undefined],
+ [Object.assign(new Error("private-rpc-message"), { code: PLUGIN_RPC_ERROR_CODES.TIMEOUT }), undefined],
+])("records known typed RPC codes without interpreting arbitrary errors", async (error, rpcCode) => {
+ const logs: string[] = [];
+ await expect(withWorkspaceRestoreDiagnostics("workspace", () => withWorkspaceRestoreStep("workspace_transfer", async () => {
+ throw preserveEnvironmentSyncOutErrorDiagnostic(error);
+ }), async line => { logs.push(line); })).rejects.toBe(error);
+ expect(getWorkspaceRestoreDiagnostic(error)?.rpcCode).toBe(rpcCode);
+ expect(getWorkspaceRestoreDiagnostic(error)?.transferStep).toBeUndefined();
+ expect(JSON.stringify(logs)).not.toContain("private-");
+});
diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
index f92e0a42bc..5f5e2c180e 100644
--- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
+++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
@@ -378,4 +378,64 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK",
expect(unrelated).not.toHaveProperty("contexts.run_failure");
expect(unrelated).not.toHaveProperty("tags.error_code");
});
+ it("sends only validated transfer and RPC evidence, isolated from unrelated captures", async () => {
+ const Sentry = sentryPackage!;
+ const events: Array<Record<string, unknown>> = [];
+ vi.stubEnv("SENTRY_DSN_BACKEND", "https://[email protected]/1");
+ vi.doMock("../peer-version-check.js", () => ({ checkExactPeerVersions: () => ({ ok: true }) }));
+ vi.doMock("@sentry/node", () => ({ ...Sentry,
+ init: (options: Record<string, unknown>) => Sentry.init({ ...options,
+ transport: () => ({ send: async () => ({}), flush: async () => true }),
+ beforeSend: (event: Record<string, unknown>) => { events.push(event); return event; },
+ }),
+ }));
+ vi.resetModules();
+ const { sentryReady, captureRunFailure, captureException } = await import("../sentry.js");
+ await sentryReady;
+ for (const known of [true, false]) {
+ const error = Object.freeze(new Error("private-transfer-message"));
+ // Worker/typed-host propagation has its own real RPC round-trip test.
+ // This narrow contract exercises the host receipt through the real Sentry
+ // transport without requiring plugin builds or dependency lifecycle scripts.
+ const evidence = {
+ transferStep: known ? "archive_create" : "private-transfer-path",
+ transferFailureKind: known ? "command_failed" : "private-transfer-command",
+ rpcCode: known ? -32002 : -12345,
+ cause: { token: "private-transfer-token" }, output: "private-transfer-output",
+ };
+ const logs: string[] = [];
+ const restore = createWorkspaceRestoreTeardown({ stagedRuntime: {
+ restoreWorkspace: progress => withWorkspaceRestoreDiagnostics("workspace", () =>
+ withWorkspaceRestoreStep("workspace_transfer", async () => { throw preserveWorkspaceRestoreErrorDiagnostic(error, { exitCode: 2 }, evidence); }), progress),
+ }, onLog: async (_stream, line) => { logs.push(line); }, startMessage: "Restoring workspace\n", failurePrefix: "Workspace restore failed" });
+ const outcome = await restore();
+ expect(outcome.ok).toBe(false);
+ if (outcome.ok) throw new Error("Expected transfer fixture failure");
+ expect(outcome.diagnostic?.rpcCode).toBe(known ? -32002 : undefined);
+ expect(JSON.stringify({ outcome, logs })).not.toContain("private-transfer-");
+ const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({ resultJson: {
+ workspaceRestoreFailure: outcome.code, workspaceRestoreDiagnostic: { ...outcome.diagnostic,
+ message: "private-transfer-persisted", cause: { token: "private-transfer-token" },
+ },
+ } } as unknown as typeof heartbeatRuns.$inferSelect, {}));
+ captureRunFailure({ taskId: "11111111-1111-4111-8111-111111111111", runId: "22222222-2222-4222-8222-222222222222",
+ errorMessage: "Workspace restore failed. Workspace files need recovery.", errorCode: "workspace_restore_failed",
+ agentAdapter: "fixture-adapter", runStatus: "failed", exitCode: 1, signal: null, diagnostics,
+ });
+ await Sentry.flush(2000);
+ const execution = (events.at(-1)?.contexts as Record<string, Record<string, unknown>>).run_execution;
+ expect(execution.workspaceRestoreTransferStep).toBe(known ? "archive_create" : undefined);
+ expect(execution.workspaceRestoreTransferFailureKind).toBe(known ? "command_failed" : undefined);
+ expect(execution.workspaceRestoreRpcCode).toBe(known ? -32002 : undefined);
+ expect(execution.workspaceRestoreExitCode).toBe(2);
+ }
+ captureException(new Error("Unrelated transfer fixture"));
+ await Sentry.flush(2000);
+ expect(events).toHaveLength(3);
+ expect(events[2]).not.toHaveProperty("contexts.run_execution");
+ expect(events[2]).not.toHaveProperty("contexts.run_failure");
+ expect(events[2]).not.toHaveProperty("tags.error_code");
+ expect(JSON.stringify(events)).not.toContain("private-transfer-");
+ });
+
});
diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts
index 0bf0d2a9eb..868e1841e1 100644
--- a/server/src/services/__tests__/run-failure-diagnostics.test.ts
+++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts
@@ -342,3 +342,33 @@ describe("run failure diagnostics", () => {
expect(collectRunFailureDiagnostics(run({ startedAt: new Date(NaN), finishedAt: new Date() }), {}).execution).not.toHaveProperty("durationMs");
});
});
+
+
+it("revalidates persisted transfer evidence and excludes arbitrary nested payloads", () => {
+ for (const known of [true, false]) {
+ const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: {
+ workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: {
+ phase: "workspace", step: "workspace_transfer", errorCode: "unknown",
+ transferStep: known ? "archive_create" : "private-command",
+ transferFailureKind: known ? "command_failed" : "private-output", rpcCode: known ? -32003 : -12345,
+ command: "private-command", cause: { message: "private-nested-message" },
+ },
+ } }), {}));
+ expect(result.execution.workspaceRestoreTransferStep).toBe(known ? "archive_create" : undefined);
+ expect(result.execution.workspaceRestoreTransferFailureKind).toBe(known ? "command_failed" : undefined);
+ expect(result.execution.workspaceRestoreRpcCode).toBe(known ? -32003 : undefined);
+ expect(JSON.stringify(result)).not.toContain("private-");
+ }
+});
+
+it("does not project transfer fields into an unrelated restore step", () => {
+ const result = collectRunFailureDiagnostics(run({ resultJson: {
+ workspaceRestoreFailure: "restore_failed", workspaceRestoreDiagnostic: {
+ phase: "workspace", step: "git_import", errorCode: "unknown",
+ transferStep: "archive_create", transferFailureKind: "command_failed", rpcCode: -32003,
+ },
+ } }), {});
+ expect(result.execution).not.toHaveProperty("workspaceRestoreTransferStep");
+ expect(result.execution).not.toHaveProperty("workspaceRestoreTransferFailureKind");
+ expect(result.execution).not.toHaveProperty("workspaceRestoreRpcCode");
+});
diff --git a/server/src/services/environment-sync-out-error.ts b/server/src/services/environment-sync-out-error.ts
index f9574f696a..64a4aa75a2 100644
--- a/server/src/services/environment-sync-out-error.ts
+++ b/server/src/services/environment-sync-out-error.ts
@@ -1,15 +1,17 @@
-import { readEnvironmentSyncErrorDiagnostic } from "@paperclipai/plugin-sdk";
+import { readEnvironmentSyncErrorDiagnostic, JsonRpcCallError, PLUGIN_RPC_ERROR_CODES } from "@paperclipai/plugin-sdk";
import { preserveWorkspaceRestoreErrorDiagnostic } from "@paperclipai/adapter-utils/workspace-restore-diagnostics";
/** Keep the original RPC failure and policy fields; add only private diagnostic evidence. */
export function preserveEnvironmentSyncOutErrorDiagnostic(error: unknown): unknown {
const diagnostic = readEnvironmentSyncErrorDiagnostic(error);
- if (diagnostic && error && typeof error === "object") {
+ const rpcCode = error instanceof JsonRpcCallError && (Object.values(PLUGIN_RPC_ERROR_CODES) as number[]).includes(error.code)
+ ? error.code : undefined;
+ if ((diagnostic || rpcCode !== undefined) && error && typeof error === "object") {
preserveWorkspaceRestoreErrorDiagnostic(error, {
- code: diagnostic.errorCode,
- status: diagnostic.httpStatus,
- exitCode: diagnostic.exitCode,
- });
+ code: diagnostic?.errorCode,
+ status: diagnostic?.httpStatus,
+ exitCode: diagnostic?.exitCode,
+ }, { ...diagnostic, rpcCode });
}
return error;
}
diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts
index 605f0b8afe..71188371e3 100644
--- a/server/src/services/run-failure-diagnostics.ts
+++ b/server/src/services/run-failure-diagnostics.ts
@@ -203,6 +203,9 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
if (diagnostic.step) execution.workspaceRestoreStep = diagnostic.step;
if (diagnostic.httpStatus !== undefined) execution.workspaceRestoreHttpStatus = diagnostic.httpStatus;
if (diagnostic.exitCode !== undefined) execution.workspaceRestoreExitCode = diagnostic.exitCode;
+ if (diagnostic.transferStep) execution.workspaceRestoreTransferStep = diagnostic.transferStep;
+ if (diagnostic.transferFailureKind) execution.workspaceRestoreTransferFailureKind = diagnostic.transferFailureKind;
+ if (diagnostic.rpcCode !== undefined) execution.workspaceRestoreRpcCode = diagnostic.rpcCode;
if (diagnostic.gitCommand) execution.workspaceRestoreGitCommand = diagnostic.gitCommand;
if (diagnostic.gitFailureKind) execution.workspaceRestoreGitFailureKind = diagnostic.gitFailureKind;
}