Skip to content

The open-source app everyone uses to manage agents at work

fix(sentry): preserve run failure stacks and diagnostic context (#14585)

Dottab3eb03fcbaaa
diff --git a/doc/observability.md b/doc/observability.md
index 8a379342a1..93513a8071 100644
--- a/doc/observability.md
+++ b/doc/observability.md
@@ -415,6 +415,37 @@ values become `unknown`. A signal such as `SIGKILL` does not establish who sent
it or prove an out-of-memory kill. These fields do not change error grouping
or run outcomes, and do not include process output or adapter result payloads.
+The shared reporter also attaches bounded diagnostic contexts for both legacy
+and native runs:
+
+- `run_execution`: runtime mode, execution stage/native phase, driver and version,
+ duration, failure phase, stop reason, error family, and timeout settings when available.
+- `adapter_failure`: selected adapter error fields such as phase, category,
+ protocol code, retryability, cause message, stack preview, HTTP status, and request ID.
+- `provider_failure`: the saved provider failure category, title, and details.
+- `run_exception_0` through `run_exception_3`: exception names, codes, HTTP
+ statuses, and request IDs for a caught exception and up to three causes.
+
+The execution and setup catch paths pass the original exception to the reporter.
+It snapshots and rebuilds only these selected fields and the original message and
+stack. Sentry receives the sanitized cause chain rather than a stack created at
+the reporting call. Saved adapter results use an available adapter stack preview;
+without one, the report has no synthetic reporting stack.
+
+Credential patterns, the current user's home path, registered run-secret values,
+and known host/runtime environment credentials are removed before truncation.
+Declared environment secret bindings are included even when their key has no
+credential-like name. Runtime values are used only for redaction and are never
+copied into an event. Stack fields are limited to 8,192 characters,
+exception messages to 2,048, provider titles to 4,096, and provider details to
+12,288. Other diagnostic strings are limited to 200 characters (adapter cause
+messages: 2,048). Truncation is marked in the text and in
+`run_execution.truncatedFields`; cyclic and deeper cause chains are marked too.
+Request/response objects, headers, environment/configuration, prompts, stdout,
+stderr, and arbitrary adapter result fields are not copied. The existing DSN
+opt-in gate and error-code/adapter fingerprint remain unchanged. These reports
+cannot recover diagnostic data that a provider or adapter discarded upstream.
+
### Browser data
The browser sends no page URL, no referrer, no user agent, and no
diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts
index 39d9e150fe..6798347819 100644
--- a/server/src/__tests__/chat-channels.integration.test.ts
+++ b/server/src/__tests__/chat-channels.integration.test.ts
@@ -1048,7 +1048,19 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
try {
await Promise.all([...fixtureServices].map((service) => service.shutdown()));
} finally {
- await retireFixtureState([...fixtureCompanies]);
+ const companyIds = [...fixtureCompanies];
+ await retireFixtureState(companyIds);
+ if (companyIds.length > 0) {
+ // Pausing an endpoint does not remove its rows from global recovery
+ // selectors. After every assertion and worker shutdown, settle leftover
+ // fixture work so later cases cannot claim its leases or retry its I/O.
+ await db.update(chatActions).set({ status: "cancelled" })
+ .where(and(inArray(chatActions.companyId, companyIds), notInArray(chatActions.status, ["processed", "cancelled"])));
+ await db.update(chatDeliveries).set({ state: "failed", nextAttemptAt: null })
+ .where(and(inArray(chatDeliveries.companyId, companyIds), inArray(chatDeliveries.state, ["received", "processing", "retry"])));
+ await db.update(chatPublications).set({ state: "cancelled", nextAttemptAt: null })
+ .where(and(inArray(chatPublications.companyId, companyIds), inArray(chatPublications.state, ["pending", "awaiting_consent", "streaming", "retry"])));
+ }
fixtureServices.clear();
fixtureCompanies.clear();
}
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index 3d95884685..f6d391e0d5 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -1565,8 +1565,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
"continuation_task_ownership_changed",
])("retains untyped continuation setup failures: %s", async (message) => {
const { runId, wakeupRequestId } = await seedQueuedIssueRunFixture();
+ const error = new Error(message, { cause: Object.assign(new Error("upstream setup failed"), { code: "ECONNRESET" }) });
const build = vi.spyOn(executionContinuation, "buildExecutionContinuation")
- .mockRejectedValueOnce(new Error(message));
+ .mockRejectedValueOnce(error);
try {
const heartbeat = heartbeatService(db);
await heartbeat.resumeQueuedRuns();
@@ -1576,6 +1577,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(await heartbeat.getRun(runId)).toMatchObject({
status: "failed", errorCode: "setup_failed", error: message,
});
+ const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]);
+ expect(report?.diagnostics).toMatchObject({
+ execution: { failurePhase: "setup" },
+ exceptions: [{ message, stack: expect.stringContaining("heartbeat-process-recovery.test.ts") }, { code: "ECONNRESET" }],
+ });
const [wakeup] = await db.select().from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, wakeupRequestId));
expect(wakeup.status).toBe("failed");
@@ -4801,6 +4807,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(failedRun?.error).toContain(
"is not installed or its plugin worker is not running",
);
+ const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]);
+ expect(report?.diagnostics).toMatchObject({
+ execution: { failurePhase: "execute" },
+ exceptions: [{ stack: expect.stringContaining("heartbeat-process-recovery.test.ts") }],
+ });
const interaction = await waitForValue(async () => {
const row = await db
@@ -5282,6 +5293,38 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(validationComment).toBeTruthy();
});
+ it.each(["throw", "result"])("redacts opaque environment-bound credentials from Sentry diagnostics: %s", async (mode) => {
+ const { companyId, agentId, runId } = await seedQueuedIssueRunFixture();
+ const svc = secretService(db);
+ const value = "opaque-runtime-credential-fixture";
+ const secret = await svc.create(companyId, {
+ name: `sentry-redaction-${randomUUID()}`, provider: "local_encrypted", value,
+ });
+ const env = { CUSTOM_BINDING: { type: "secret_ref", secretId: secret.id, version: "latest" } };
+ await svc.syncEnvBindingsForTarget(companyId, { targetType: "agent", targetId: agentId }, env);
+ await db.update(agents).set({ adapterConfig: { env } }).where(eq(agents.id, agentId));
+ mockAdapterExecute.mockImplementationOnce(async (input) => {
+ expect(input.config.env.CUSTOM_BINDING).toBe(value);
+ const message = `upstream rejected ${value}`;
+ if (mode === "throw") throw new Error(message, { cause: new Error(message) });
+ return {
+ exitCode: 1, signal: null, timedOut: false, errorMessage: message, errorCode: "adapter_failed",
+ errorMeta: { causeMessage: message },
+ resultJson: { terminalSessionFailure: { category: "service", details: message } },
+ };
+ });
+ const heartbeat = heartbeatService(db);
+ await heartbeat.resumeQueuedRuns();
+ await waitForRunToSettle(heartbeat, runId, 5_000);
+ await heartbeat.waitForRunExecutionDrain(runId);
+ const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]);
+ expect(report).toBeDefined();
+ expect(report.errorMessage).toContain("upstream rejected");
+ expect(JSON.stringify(report)).not.toContain(value);
+ if (mode === "throw") expect(report.diagnostics.exceptions).toHaveLength(2);
+ else expect(report.diagnostics.provider.category).toBe("service");
+ });
+
it("blocks before dispatch when a declared secret ref has no binding instead of emitting an opaque setup failure", async () => {
const { companyId, agentId, runId, issueId } =
await seedQueuedIssueRunFixture();
diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
index 5cfe758e2b..11b3c259f5 100644
--- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
+++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts
@@ -1,5 +1,7 @@
import { createRequire } from "node:module";
import { afterEach, describe, expect, it, vi } from "vitest";
+import { collectRunFailureDiagnostics, sanitizeRunFailureDiagnostics } from "../services/run-failure-diagnostics.js";
+import type { heartbeatRuns } from "@paperclipai/db";
// Sentry is an optional peer. When installed, exercise the real SDK with an
// in-memory transport, including its context behavior without an OTel manager.
@@ -76,7 +78,17 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK",
exitCode: null,
signal: "SIGKILL",
};
- captureRunFailure(first);
+ const original = Object.assign(new Error(first.errorMessage, { cause: Object.assign(new Error("upstream connection reset"), {
+ code: "ECONNRESET", requestId: "request-123", status: 503,
+ stack: "Error: upstream connection reset\n at socketRead (/app/provider.js:19:7)",
+ response: { body: "private-response" },
+ }) }), { stack: "Error: first run failed\n at originalAdapter (/app/adapter.js:42:7)", request: { headers: "private-headers" } });
+ const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({
+ runtimeMode: "legacy", resultJson: { terminalSessionFailure: {
+ category: "service", title: "Provider failed", details: "d".repeat(9000),
+ } },
+ } as unknown as typeof heartbeatRuns.$inferSelect, { error: original, phase: "execute" }));
+ captureRunFailure({ ...first, diagnostics });
await Promise.resolve();
captureException(new Error("unrelated database error"));
captureRunFailure(second);
@@ -91,7 +103,7 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK",
expect(events).toHaveLength(5);
const captured = (message: string) => events.find((event) =>
- (event.exception as { values: Array<{ value: string }> }).values[0]?.value === message,
+ (event.exception as { values: Array<{ value: string }> }).values.some((value) => value.value === message),
);
for (const run of [first, second]) {
expect(captured(run.errorMessage)).toMatchObject({
@@ -108,10 +120,25 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK",
fingerprint: [first.errorCode, first.agentAdapter],
});
expect(JSON.stringify(events)).not.toContain("private-signal-payload");
+ const firstCapture = captured(first.errorMessage)!;
+ const exceptions = (firstCapture.exception as { values: Array<Record<string, unknown>> }).values;
+ expect(exceptions).toHaveLength(2);
+ expect(JSON.stringify(exceptions)).toContain("originalAdapter");
+ expect(JSON.stringify(exceptions)).toContain("socketRead");
+ expect(JSON.stringify(exceptions)).not.toContain("captureRunFailure");
+ expect(JSON.stringify(captured(second.errorMessage)?.exception)).not.toContain("captureRunFailure");
+ expect(firstCapture).toMatchObject({ contexts: {
+ provider_failure: { category: "service", details: "d".repeat(9000) },
+ run_exception_1: { code: "ECONNRESET", requestId: "request-123", status: 503 },
+ } });
+ expect(JSON.stringify(events)).not.toContain("private-");
for (const message of ["unrelated database error", "unrelated filesystem error"]) {
const event = captured(message);
expect(event).toBeDefined();
expect(event).not.toHaveProperty("contexts.run_failure");
+ for (const context of ["run_execution", "adapter_failure", "provider_failure", "run_exception_0", "run_exception_1"]) {
+ expect(event).not.toHaveProperty(`contexts.${context}`);
+ }
expect(event).not.toHaveProperty("fingerprint");
for (const tag of ["run_id", "task_id", "error_code", "agent_adapter", "run_status"]) {
expect(event).not.toHaveProperty(`tags.${tag}`);
diff --git a/server/src/__tests__/run-failure-sentry.test.ts b/server/src/__tests__/run-failure-sentry.test.ts
index d5285698a4..9ba4c7467f 100644
--- a/server/src/__tests__/run-failure-sentry.test.ts
+++ b/server/src/__tests__/run-failure-sentry.test.ts
@@ -215,6 +215,31 @@ describe("captureRunFailure", () => {
expect((received as Error).message).toBe("boom");
});
+ it("uses sanitized original stacks and causes instead of the reporting stack", async () => {
+ const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
+ sentryModule.captureRunFailure(baseEvent({ diagnostics: {
+ execution: { runtimeMode: "native", failurePhase: "setup" },
+ adapter: {}, provider: {}, truncatedFields: [],
+ exceptions: [
+ { name: "TypeError", message: "setup failed", stack: "TypeError: setup failed\n at originalSetup (/app/setup.js:42:7)" },
+ { name: "Error", message: "connection reset", stack: "Error: connection reset\n at socketRead (/app/network.js:9:4)", code: "ECONNRESET", requestId: "req-123" },
+ ],
+ } }));
+ const [error, context] = captureException.mock.calls[0]!;
+ expect(error).toBeInstanceOf(Error);
+ expect(error).toMatchObject({ name: "TypeError", message: "setup failed", cause: { message: "connection reset" } });
+ expect((error as Error).stack).toContain("originalSetup");
+ expect((error as Error).stack).not.toContain("captureRunFailure");
+ expect(context?.contexts.run_exception_1).toMatchObject({ code: "ECONNRESET", requestId: "req-123" });
+ expect(context?.contexts.run_execution).toMatchObject({ failurePhase: "setup" });
+ });
+
+ it("does not invent a reporter stack for a saved result with no original exception", async () => {
+ const { sentryModule, captureException } = await importFreshSentryWithGateOpen();
+ sentryModule.captureRunFailure(baseEvent());
+ expect((captureException.mock.calls[0]![0] as Error).stack).toBeUndefined();
+ });
+
it("does not throw and captures nothing when the gate is closed", async () => {
vi.resetModules();
const sentryModule = await import("../sentry.js");
diff --git a/server/src/sentry.ts b/server/src/sentry.ts
index 1a9c30f464..ee7f28f49d 100644
--- a/server/src/sentry.ts
+++ b/server/src/sentry.ts
@@ -56,6 +56,7 @@
// `instrumentation.ts`.
import os from "node:os";
+import type { RunFailureDiagnostics } from "./services/run-failure-diagnostics.js";
import { readBuildCommit } from "./build-commit.js";
import { checkExactPeerVersions } from "./peer-version-check.js";
import { resolveSentryDsns } from "./sentry-dsn.js";
@@ -132,6 +133,8 @@ export interface RunFailureEvent {
agentAdapter: string;
/** The run status that triggered this report. */
runStatus: RunFailureStatus;
+ /** Bounded, redacted diagnostics selected by the run failure reporter. */
+ diagnostics?: RunFailureDiagnostics;
/** Recorded process exit evidence, when available. Validated before capture. */
exitCode?: number | null;
signal?: string | null;
@@ -174,7 +177,33 @@ export function captureRunFailure(event: RunFailureEvent): void {
// Sentry's async scope isolation is absent when OTel setup is skipped.
// A withScope mutation can then persist into unrelated later captures.
// Pass these fields on this event only; do not mutate the ambient scope.
- handle.captureException(new Error(event.errorMessage), {
+ const diagnostics = event.diagnostics;
+ const contexts: Record<string, Record<string, unknown>> = {};
+ if (diagnostics) {
+ contexts.run_execution = { ...diagnostics.execution, truncatedFields: diagnostics.truncatedFields };
+ if (Object.keys(diagnostics.adapter).length) contexts.adapter_failure = diagnostics.adapter;
+ if (Object.keys(diagnostics.provider).length) contexts.provider_failure = diagnostics.provider;
+ diagnostics.exceptions.forEach(({ name, code, status, requestId }, index) => {
+ contexts[`run_exception_${index}`] = { name, code, status, requestId };
+ });
+ }
+ // Rebuild only sanitized exception fields. Passing a raw SDK Error can
+ // serialize its request/response, headers, or other enumerable properties.
+ let cause: Error | undefined;
+ for (const entry of [...(diagnostics?.exceptions ?? [])].reverse()) {
+ const error: Error = new Error(entry.message ?? event.errorMessage, cause ? { cause } : undefined);
+ error.name = entry.name ?? "Error";
+ error.stack = entry.stack;
+ cause = error;
+ }
+ const exception = cause ?? new Error(event.errorMessage);
+ if (!cause) {
+ // A saved adapter result is not a thrown Error. Do not pretend that the
+ // reporter's own stack is the failure location.
+ exception.stack = typeof diagnostics?.adapter.stackPreview === "string"
+ ? diagnostics.adapter.stackPreview : undefined;
+ }
+ handle.captureException(exception, {
tags: {
run_id: event.runId,
task_id: event.taskId,
@@ -183,6 +212,7 @@ export function captureRunFailure(event: RunFailureEvent): void {
run_status: event.runStatus,
},
contexts: {
+ ...contexts,
run_failure: {
taskId: event.taskId,
runId: event.runId,
diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts
new file mode 100644
index 0000000000..838d63f8d0
--- /dev/null
+++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts
@@ -0,0 +1,95 @@
+import { describe, expect, it } from "vitest";
+import type { heartbeatRuns } from "@paperclipai/db";
+import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js";
+
+type Run = typeof heartbeatRuns.$inferSelect;
+const run = (overrides: Partial<Run> = {}) => ({ resultJson: null, ...overrides }) as Run;
+const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error });
+
+describe("run failure diagnostics", () => {
+ it("selects declared environment secrets under opaque names and common credential keys", () => {
+ expect(collectRunFailureSecretValues({
+ CUSTOM_BINDING: "bound-opaque-value", ACCESS_TOKEN: "plain-opaque-value", REGION: "us-east-1", EMPTY_KEY: "",
+ }, ["CUSTOM_BINDING"])).toEqual(["bound-opaque-value", "plain-opaque-value", "us-east-1"]);
+ expect(collectRunFailureSecretValues({ PATH: "/usr/bin", CUSTOM_BINDING: "opaque" }, [], true)).toEqual(["opaque"]);
+ expect(collectRunFailureSecretValues({ PATH: "secret-override" }, ["PATH"], true)).toEqual(["secret-override"]);
+ });
+
+ it("redacts encoded credentials and URL passwords without expanding replacement markers", () => {
+ const secret = 'opaque/"credential';
+ const values = collectRunFailureSecretValues({ DATABASE_URL: `postgres://user:${encodeURIComponent(secret)}@example/db` });
+ const result = redactRunFailureSecretValues({ text: `${secret} ${encodeURIComponent(secret)} ${JSON.stringify(secret)}`, status: 503 }, [...values, "REDACTED"]);
+ expect(result.text).not.toContain("opaque");
+ expect(result.status).toBe(503);
+ expect(result.text).not.toContain("***[REDACTED]");
+ });
+
+ it("preserves known boolean and numeric settings without exempting unknown or declared secrets", () => {
+ const env = { PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_DB_BACKUP_RETENTION_DAYS: "1" };
+ expect(collectRunFailureSecretValues(env, [], true)).toEqual([]);
+ expect(collectRunFailureSecretValues(env, ["PAPERCLIP_DB_BACKUP_ENABLED"], true)).toEqual(["false"]);
+ expect(collectRunFailureSecretValues({ CUSTOM_BINDING: "1" }, [], true)).toEqual(["1"]);
+ expect(collectRunFailureSecretValues({ PAPERCLIP_DB_BACKUP_ENABLED: "opaque" }, [], true)).toEqual(["opaque"]);
+ });
+ it("preserves generic exceptions, numeric codes and nested network causes", () => {
+ const root = Object.assign(new Error("network failed"), { code: "ECONNRESET", statusCode: 502, request_id: "req-123" });
+ const outer = Object.assign(new TypeError("adapter failed", { cause: root }), { code: -32000 });
+ const result = sanitizeRunFailureDiagnostics(collect(outer));
+ expect(result.exceptions).toMatchObject([
+ { name: "TypeError", message: "adapter failed", code: "-32000" },
+ { name: "Error", message: "network failed", code: "ECONNRESET", status: 502, requestId: "req-123" },
+ ]);
+ expect(result.exceptions[0].stack).toContain("run-failure-diagnostics.test.ts");
+ });
+
+ it("bounds cyclic, deep, and string causes without inspecting arbitrary error objects", () => {
+ const cyclic = new Error("cycle"); cyclic.cause = cyclic;
+ expect(collect(cyclic)).toMatchObject({ exceptions: [{ message: "cycle" }], truncatedFields: ["exceptions.cycle"] });
+ let error = new Error("leaf");
+ for (let i = 0; i < 10; i++) error = new Error(`level-${i}`, { cause: error });
+ expect(collect(error).exceptions).toHaveLength(4);
+ expect(collect(error).truncatedFields).toContain("exceptions.depth");
+ expect(collect(new Error("outer", { cause: "string cause" })).exceptions).toHaveLength(2);
+ const getter = Object.defineProperty({ message: "valid message" }, "response", { get: () => { throw new Error("must not read"); } });
+ expect(collect(getter).exceptions).toEqual([{ message: "valid message" }]);
+ expect(collect({ arbitrary: "private payload" }).exceptions).toEqual([]);
+ expect(collect(Object.defineProperty({}, "message", { get: () => { throw new Error("bad getter"); } })).exceptions).toEqual([]);
+ });
+
+ it("redacts credential forms before truncation in all diagnostic text", () => {
+ const failure = collectRunFailureDiagnostics(run({ resultJson: {
+ terminalSessionFailure: { category: "service", title: 'password="secret-password"', details: 'Authorization: Bearer secret-bearer' },
+ } }), { error: Object.assign(new Error('apiKey="secret-api-key"'), { cause: new Error('token="secret-token"') }), adapterErrorMeta: { causeMessage: 'password="secret-password"' } });
+ const result = sanitizeRunFailureDiagnostics(failure);
+ expect(JSON.stringify(result)).not.toContain("secret-");
+ expect(sanitizeRunFailureText('x'.repeat(90) + ' password="secret-at-cut-boundary"', 110)).not.toContain("secret-at");
+ expect(sanitizeRunFailureText('pass\x1b[31mword="secret-colored"', 200)).not.toContain("secret-colored");
+ });
+
+ it("reports truncation, preserves UTF-16 pairs, and keeps the total payload bounded", () => {
+ const huge = "πŸ˜€".repeat(100_000);
+ let error = Object.assign(new Error(huge), { stack: huge });
+ for (let i = 0; i < 5; i++) error = Object.assign(new Error(huge, { cause: error }), { stack: huge });
+ const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: {
+ terminalSessionFailure: { category: "service", title: huge, details: huge, truncatedFields: ["details", "private-field"] },
+ } }), { error }));
+ expect(String(result.provider.title).length).toBeLessThanOrEqual(4096);
+ expect(String(result.provider.details).length).toBeLessThanOrEqual(12288);
+ expect(JSON.stringify(result)).not.toMatch(/[\ud800-\udfff]/u);
+ expect(result.truncatedFields).toContain("provider.details");
+ expect(result.truncatedFields).toContain("exceptions.0.stack");
+ expect(result.truncatedFields).not.toContain("provider.private-field");
+ expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan(150_000);
+ });
+
+ it("keeps scalar native and legacy execution evidence and rejects non-finite durations", () => {
+ const result = collectRunFailureDiagnostics(run({
+ runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3",
+ startedAt: new Date(1000), finishedAt: new Date(8500),
+ resultJson: { stopReason: "failed", timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream", summary: "private prose" },
+ }), { phase: "execute" });
+ expect(result.execution).toMatchObject({ runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3", durationMs: 7500, timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream" });
+ expect(JSON.stringify(result)).not.toContain("private prose");
+ expect(collectRunFailureDiagnostics(run({ startedAt: new Date(NaN), finishedAt: new Date() }), {}).execution).not.toHaveProperty("durationMs");
+ });
+});
diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.ts
index 9b32245e76..2b7e184e48 100644
--- a/server/src/services/__tests__/run-failure-report.test.ts
+++ b/server/src/services/__tests__/run-failure-report.test.ts
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import os from "node:os";
-import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db";
import {
getEmbeddedPostgresTestSupport,
@@ -9,6 +9,11 @@ import {
const mockCaptureRunFailure = vi.hoisted(() => vi.fn());
const mockRedactCurrentUserText = vi.hoisted(() => vi.fn());
+const mockResolveSecret = vi.hoisted(() => vi.fn());
+
+vi.mock("../../secrets/provider-registry.js", () => ({
+ getSecretProvider: () => ({ resolveVersion: mockResolveSecret }),
+}));
vi.mock("../../sentry.js", () => ({
captureRunFailure: mockCaptureRunFailure,
@@ -23,8 +28,7 @@ vi.mock("../../log-redaction.js", async (importOriginal) => {
});
import { reportRunFailure, waitForPendingRunFailureReports } from "../run-failure-report.js";
-import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../../redaction.js";
-import { redactCurrentUserText } from "../../log-redaction.js";
+import { REDACTED_EVENT_VALUE } from "../../redaction.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
@@ -34,14 +38,29 @@ describeEmbeddedPostgres("reportRunFailure", () => {
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let companyId!: string;
let agentId!: string;
+ let inheritedEnv: NodeJS.ProcessEnv;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-failure-report-");
db = createDb(tempDb.connectionString);
}, 20_000);
+ beforeEach(() => {
+ // Unknown host values intentionally count as secrets, including short values
+ // such as "1". Keep these fixtures independent of the developer/CI environment
+ // and add secret values explicitly in the tests that exercise redaction.
+ inheritedEnv = process.env;
+ process.env = Object.fromEntries(
+ ["PATH", "HOME", "USER", "USERNAME", "LOGNAME", "USERPROFILE", "TMPDIR", "TEMP", "TMP"]
+ .flatMap((key) => inheritedEnv[key] === undefined ? [] : [[key, inheritedEnv[key]]]),
+ );
+ });
+
afterEach(async () => {
vi.clearAllMocks();
+ vi.unstubAllEnvs();
+ process.env = inheritedEnv;
+ await db.delete(heartbeatRuns);
await db.delete(agents);
await db.delete(companies);
});
@@ -141,6 +160,98 @@ describeEmbeddedPostgres("reportRunFailure", () => {
}
});
+ it("reports selected provider diagnostics and the original cause without copying arbitrary payloads", async () => {
+ await seedCompanyAndAgent();
+ const cause = Object.assign(new Error("provider unreachable"), {
+ code: "ECONNRESET", requestId: "request-123", status: 503,
+ response: { body: "private-response" },
+ });
+ const error = new Error("adapter threw", { cause });
+ const run = buildRun({
+ runtimeMode: "legacy",
+ executionStage: "execute",
+ error: "adapter threw",
+ stderrExcerpt: "private-stderr",
+ stdoutExcerpt: "private-stdout",
+ contextSnapshot: { prompt: "private-prompt" },
+ resultJson: {
+ terminalSessionFailure: { category: "service", title: "Provider unavailable", details: "request-123 failed", raw: "private-provider-raw" },
+ timeoutFired: false, summary: "private-summary", env: { SECRET: "private-env" },
+ },
+ });
+ await reportRunFailure(db, run, { error, phase: "execute", adapterErrorMeta: { phase: "turn", retryable: true, response: "private-adapter-response" } });
+ const captured = mockCaptureRunFailure.mock.calls[0][0];
+ expect(captured.diagnostics).toMatchObject({
+ execution: { runtimeMode: "legacy", executionStage: "execute", failurePhase: "execute", timeoutFired: false },
+ adapter: { phase: "turn", retryable: true },
+ provider: { category: "service", title: "Provider unavailable", details: "request-123 failed" },
+ exceptions: [{ message: "adapter threw" }, { message: "provider unreachable", code: "ECONNRESET", status: 503, requestId: "request-123" }],
+ });
+ expect(captured.diagnostics.exceptions[0].stack).toContain("run-failure-report.test.ts");
+ expect(JSON.stringify(captured)).not.toContain("private-");
+ expect(error.cause).toBe(cause);
+ });
+
+ it.each([false, true])("redacts registered run secrets and fails closed when resolution fails: %s", async (fails) => {
+ await seedCompanyAndAgent();
+ const secret = "opaque-registered-value";
+ const run = buildRun({
+ error: `connection failed: ${secret}`,
+ contextSnapshot: { paperclipSecretRedactions: [{ fingerprintSha256: "fixture", material: { encrypted: "fixture" } }] },
+ resultJson: { terminalSessionFailure: { category: "service", details: `upstream rejected ${secret}` } },
+ });
+ await db.insert(heartbeatRuns).values(run);
+ if (fails) mockResolveSecret.mockRejectedValueOnce(new Error("fixture resolution failed"));
+ else mockResolveSecret.mockResolvedValueOnce(secret);
+
+ await expect(reportRunFailure(db, run, {
+ error: new Error(`failed ${secret}`, { cause: new Error(`cause ${secret}`) }),
+ adapterErrorMeta: { causeMessage: `adapter ${secret}` },
+ })).resolves.toBeUndefined();
+ expect(mockResolveSecret).toHaveBeenCalledTimes(1);
+ if (fails) {
+ expect(mockCaptureRunFailure).not.toHaveBeenCalled();
+ } else {
+ expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
+ const captured = mockCaptureRunFailure.mock.calls[0][0];
+ expect(JSON.stringify(captured)).not.toContain(secret);
+ expect(captured.diagnostics.provider.details).toContain(REDACTED_EVENT_VALUE);
+ expect(captured.diagnostics.exceptions[1].message).toContain(REDACTED_EVENT_VALUE);
+ }
+ });
+
+ it("redacts runtime and host environment secret values even without a persisted registry", async () => {
+ await seedCompanyAndAgent();
+ const runtimeSecret = "opaque-runtime-value";
+ const hostSecret = "opaque-host-value";
+ vi.stubEnv("FIXTURE_CUSTOM_VALUE", hostSecret);
+ const text = `connection failed: ${runtimeSecret} ${hostSecret}`;
+ const run = buildRun({ error: text, contextSnapshot: null,
+ resultJson: { terminalSessionFailure: { details: text } },
+ });
+ await reportRunFailure(db, run, {
+ error: new Error(text, { cause: new Error(text) }),
+ adapterErrorMeta: { causeMessage: text, stackPreview: text },
+ secretValues: [runtimeSecret],
+ });
+ expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
+ const captured = mockCaptureRunFailure.mock.calls[0][0];
+ expect(JSON.stringify(captured)).not.toContain(runtimeSecret);
+ expect(JSON.stringify(captured)).not.toContain(hostSecret);
+ expect(captured.diagnostics.exceptions).toHaveLength(2);
+ expect(captured).not.toHaveProperty("secretValues");
+ });
+
+ it("still redacts short values from unknown host settings", async () => {
+ await seedCompanyAndAgent();
+ vi.stubEnv("FIXTURE_CUSTOM_VALUE", "1");
+
+ await reportRunFailure(db, buildRun());
+
+ expect(mockCaptureRunFailure.mock.calls[0][0].errorMessage)
+ .toBe(`the provider process exited with code ${REDACTED_EVENT_VALUE}`);
+ });
+
it("captures nothing for succeeded, cancelled, and interrupted", async () => {
await seedCompanyAndAgent();
for (const status of ["succeeded", "cancelled", "interrupted"] as const) {
@@ -259,9 +370,10 @@ describeEmbeddedPostgres("reportRunFailure", () => {
await reportRunFailure(db, run);
- const expectedErrorMessage = redactSensitiveText(redactCurrentUserText(rawError));
const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
- expect(errorMessage).toBe(expectedErrorMessage);
+ // Environment-value redaction can fully mask the username before the
+ // current-user redactor applies its partial mask. Both remove the home path.
+ expect(errorMessage).toContain("workspace/report.log");
expect(errorMessage).not.toContain(homeDir);
});
@@ -276,10 +388,13 @@ describeEmbeddedPostgres("reportRunFailure", () => {
const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
expect(errorMessage).toHaveLength(MAX_ERROR_MESSAGE_LENGTH);
- expect(errorMessage).toBe("x".repeat(MAX_ERROR_MESSAGE_LENGTH));
+ expect(errorMessage).toBe("x".repeat(MAX_ERROR_MESSAGE_LENGTH - 12) + "\n[truncated]");
});
- it("does not change a short error message", async () => {
+ it("preserves a short error message with known public host settings", async () => {
+ vi.stubEnv("PAPERCLIP_DB_BACKUP_ENABLED", "false");
+ vi.stubEnv("PAPERCLIP_DB_BACKUP_RETENTION_DAYS", "1");
+ vi.stubEnv("GITHUB_RUN_ATTEMPT", "1");
await seedCompanyAndAgent();
const shortError = "the provider process exited with code 1";
const run = buildRun({ status: "failed", error: shortError });
@@ -326,7 +441,7 @@ describeEmbeddedPostgres("reportRunFailure", () => {
const { errorCode } = mockCaptureRunFailure.mock.calls[0][0];
expect(errorCode).toHaveLength(MAX_ERROR_CODE_LENGTH);
- expect(errorCode).toBe("y".repeat(MAX_ERROR_CODE_LENGTH));
+ expect(errorCode).toBe("y".repeat(MAX_ERROR_CODE_LENGTH - 12) + "\n[truncated]");
});
it("sends errorCode null unchanged when the run holds no error code", async () => {
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 2f86e242fe..490101f2fd 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -326,6 +326,7 @@ import {
emitAgentTaskRunById,
} from "./agent-task-run-telemetry.js";
import { reportRunFailure } from "./run-failure-report.js";
+import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";
import { companySkillService } from "./company-skills.js";
import { budgetService, type BudgetEnforcementScope } from "./budgets.js";
import { secretService, type MissingRuntimeBinding } from "./secrets.js";
@@ -12754,12 +12755,13 @@ export function heartbeatService(
function emitTerminalAgentTaskRun(
updated: typeof heartbeatRuns.$inferSelect,
previousStatus: string | null,
+ failureReport?: RunFailureReportOptions,
) {
if (!isHeartbeatRunTerminalStatus(updated.status)) return;
if (previousStatus === updated.status) return;
clearHeartbeatRunRuntimeStatus(updated.id);
void emitAgentTaskRun(db, updated);
- void reportRunFailure(db, updated);
+ void reportRunFailure(db, updated, failureReport);
}
async function setRunStatus(
@@ -12833,8 +12835,9 @@ export function heartbeatService(
runId: string,
status: string,
patch?: Partial<typeof heartbeatRuns.$inferInsert>,
+ failureReport?: RunFailureReportOptions,
) {
- return setRunStatusFromLive(runId, status, ["running"], patch);
+ return setRunStatusFromLive(runId, status, ["running"], patch, failureReport);
}
// Move a run to a new status only when its current status is one of
@@ -12847,6 +12850,7 @@ export function heartbeatService(
status: string,
fromStatuses: string[],
patch?: Partial<typeof heartbeatRuns.$inferInsert>,
+ failureReport?: RunFailureReportOptions,
) {
// fromStatuses can name a terminal status as its own source (for example,
// an idempotent "still failed" patch), so the write below is not always a
@@ -12916,7 +12920,7 @@ export function heartbeatService(
payload: buildHeartbeatRunStatusLiveEventPayload(updated),
});
publishRunLifecyclePluginEvent(updated);
- emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null);
+ emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport);
return { run: updated, updated: true as const };
}
@@ -20238,6 +20242,7 @@ export function heartbeatService(
ReturnType<typeof traceStore.prepare>
> | null = null;
let providerTraceFinalized = false;
+ let readFailureReportSecrets: () => string[] = () => [];
try {
const agent = await getAgent(run.agentId);
@@ -21414,6 +21419,7 @@ export function heartbeatService(
secretsSvc,
trustPreset,
});
+ readFailureReportSecrets = () => collectRunFailureSecretValues(resolvedConfig.env, secretKeys);
if (aiBinding) {
try {
managedAiRuntime = await prepareManagedAiRuntime(db, { companyId: agent.companyId, agentId: agent.id, responsibleUserId, adapterType: agent.adapterType, binding: aiBinding, config: resolvedConfig });
@@ -21504,6 +21510,11 @@ export function heartbeatService(
// Always replace this runtime-only field; caller wake data cannot supply skills.
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
let runtimeConfig: Record<string, unknown> = connectorDelivery.config;
+ const resolvedFailureSecrets = readFailureReportSecrets();
+ readFailureReportSecrets = () => [
+ ...resolvedFailureSecrets,
+ ...collectRunFailureSecretValues(runtimeConfig.env, secretKeys),
+ ];
const latestAgentConfigRevision = await getLatestAgentConfigRevision(
agent.companyId,
agent.id,
@@ -25191,6 +25202,7 @@ export function heartbeatService(
run.id,
status,
finalRunPatch,
+ { adapterErrorMeta: adapterResult.errorMeta, secretValues: readFailureReportSecrets() },
);
let persistedRun: typeof heartbeatRuns.$inferSelect | null =
persistedRunWrite.run;
@@ -25830,7 +25842,7 @@ export function heartbeatService(
logBytes: logSummary?.bytes,
logSha256: logSummary?.sha256,
logCompressed: logSummary?.compressed ?? false,
- });
+ }, { error: err, phase: "execute", secretValues: readFailureReportSecrets() });
if (
!failedRunWrite.updated &&
!(
@@ -26084,7 +26096,7 @@ export function heartbeatService(
: setupFailureResultJson
? { resultJson: setupFailureResultJson }
: {}),
- }).catch(() => ({ run: null, updated: false as const }));
+ }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets() }).catch(() => ({ run: null, updated: false as const }));
if (!setupFailureWrite.updated) {
logger.info(
{
diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts
new file mode 100644
index 0000000000..86197fc1a1
--- /dev/null
+++ b/server/src/services/run-failure-diagnostics.ts
@@ -0,0 +1,227 @@
+import type { heartbeatRuns } from "@paperclipai/db";
+import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction";
+import { redactCurrentUserText } from "../log-redaction.js";
+import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";
+
+type Run = typeof heartbeatRuns.$inferSelect;
+type Context = Record<string, string | number | boolean>;
+
+export interface RunFailureReportOptions {
+ /** The caught exception, before callers flatten it to a message. */
+ error?: unknown;
+ /** Structured adapter error metadata; only known diagnostic keys are read. */
+ adapterErrorMeta?: unknown;
+ phase?: "setup" | "execute";
+ /** Runtime-only values to redact; never included in the captured event. */
+ secretValues?: readonly string[];
+}
+
+export interface RunFailureException {
+ name?: string;
+ message?: string;
+ stack?: string;
+ code?: string;
+ status?: number;
+ requestId?: string;
+}
+
+export interface RunFailureDiagnostics {
+ execution: Context;
+ adapter: Context;
+ provider: Context;
+ exceptions: RunFailureException[];
+ truncatedFields: string[];
+}
+
+function read(value: unknown, key: string): unknown {
+ if (!value || typeof value !== "object") return undefined;
+ try {
+ return (value as Record<string, unknown>)[key];
+ } catch {
+ return undefined;
+ }
+}
+
+function scalars(value: unknown, fields: readonly string[]): Context {
+ const result: Context = {};
+ for (const field of fields) {
+ const entry = read(value, field);
+ if (
+ typeof entry === "string" || typeof entry === "boolean" ||
+ (typeof entry === "number" && Number.isFinite(entry))
+ ) result[field] = entry;
+ }
+ return result;
+}
+
+// Unknown inherited values may contain credentials under arbitrary names.
+// Explicitly configured values remain private, including overrides of these keys.
+const PUBLIC_ENV_KEYS = new Set([
+ "PATH", "PATHEXT", "SYSTEMROOT", "WINDIR", "COMSPEC", "HOME", "USERPROFILE",
+ "HOMEDRIVE", "HOMEPATH", "USER", "USERNAME", "LOGNAME", "SHELL", "LANG",
+ "LANGUAGE", "LC_ALL", "LC_CTYPE", "TZ", "TMPDIR", "TEMP", "TMP", "NODE_ENV",
+ "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME",
+ "PWD", "OLDPWD", "INIT_CWD", "SHLVL", "_", "TERM", "TERM_PROGRAM", "TERM_PROGRAM_VERSION", "COLORTERM",
+ "NPM_LIFECYCLE_EVENT", "NPM_COMMAND", "NPM_EXEC_PATH", "NPM_EXECPATH", "NPM_NODE_EXECPATH",
+ "NPM_PACKAGE_NAME", "NPM_PACKAGE_VERSION", "NPM_PACKAGE_JSON", "NPM_CONFIG_USER_AGENT",
+ "XPC_SERVICE_NAME", "VITEST_POOL_ID", "VITEST_WORKER_ID",
+ "PAPERCLIP_AGENT_ID", "PAPERCLIP_COMPANY_ID", "PAPERCLIP_RUN_ID", "PAPERCLIP_TASK_ID",
+]);
+const PUBLIC_BOOLEAN_ENV_KEYS = new Set([
+ "OPENCODE_ALLOW_ALL_MODELS", "CLAUDE_CODE_USE_BEDROCK", "GOOGLE_GENAI_USE_GCA",
+ "CI", "NO_COLOR", "FORCE_COLOR",
+ "VITEST", "MALLOCNANOZONE", "CODEX_CI", "CODEX_SHELL", "CODEX_SAGE_BACKFILL_TRACKER_TAB_REUSE",
+ "DEV", "PROD", "SSR", "PAPERCLIP_REQUIRE_SENTRY_TEST_SDK",
+ "PAPERCLIP_DB_BACKUP_ENABLED", "PAPERCLIP_AUTH_DISABLE_SIGN_UP", "PAPERCLIP_ENABLE_COMPANY_DELETION",
+ "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "PAPERCLIP_SECRETS_STRICT_MODE",
+ "SERVE_UI", "PAPERCLIP_UI_DEV_MIDDLEWARE", "HEARTBEAT_SCHEDULER_ENABLED", "PAPERCLIP_ANNOUNCEMENTS_ENABLED",
+ "GITHUB_ACTIONS", "GITHUB_REF_PROTECTED", "RUNNER_DEBUG", "ACTIONS_RUNNER_DEBUG", "ACTIONS_STEP_DEBUG",
+]);
+const PUBLIC_NUMBER_ENV_KEYS = new Set([
+ "PORT", "PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES", "PAPERCLIP_DB_BACKUP_RETENTION_DAYS",
+ "PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "HEARTBEAT_SCHEDULER_INTERVAL_MS",
+ "GITHUB_RUN_ATTEMPT", "GITHUB_RUN_NUMBER", "GITHUB_RUN_ID", "GITHUB_RETENTION_DAYS",
+]);
+
+/** Include declared bindings and unknown values, not just credential-like keys. */
+export function collectRunFailureSecretValues(
+ env: unknown, secretKeys: Iterable<string> = [], inherited = false,
+): string[] {
+ if (!env || typeof env !== "object") return [];
+ const strings = Object.fromEntries(Object.entries(env).filter(
+ (entry): entry is [string, string] => typeof entry[1] === "string",
+ ));
+ const declared = new Set(secretKeys);
+ const values: string[] = [];
+ for (const [key, value] of Object.entries(strings)) {
+ if (!value) continue;
+ const publicSetting = (inherited && (
+ PUBLIC_ENV_KEYS.has(key.toUpperCase()) ||
+ (PUBLIC_NUMBER_ENV_KEYS.has(key.toUpperCase()) && /^\d+$/.test(value)) ||
+ (key === "BASE_URL" && value === "/") ||
+ (key === "MODE" && /^(?:test|development|production)$/.test(value))
+ )) ||
+ (PUBLIC_BOOLEAN_ENV_KEYS.has(key.toUpperCase()) && /^(?:0|1|true|false)$/i.test(value));
+ if (declared.has(key) || !publicSetting) values.push(value);
+ try {
+ const password = new URL(value).password;
+ if (password) values.push(value, password, decodeURIComponent(password));
+ } catch { /* ordinary environment values are not URLs */ }
+ }
+ return [...new Set(values)].sort((a, b) => b.length - a.length);
+}
+
+/** Redact literal, JSON-escaped and URI-encoded forms without rewriting markers. */
+export function redactRunFailureSecretValues<T>(input: T, values: readonly string[]): T {
+ const forms = [...new Set(values.filter(Boolean).flatMap((value) => {
+ const forms = [value, JSON.stringify(value).slice(1, -1)];
+ try { forms.push(encodeURIComponent(value)); } catch { /* malformed Unicode */ }
+ return forms;
+ }))].sort((a, b) => b.length - a.length);
+ if (forms.length === 0) return input;
+ const pattern = new RegExp(forms.map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"), "gu");
+ const redact = (value: unknown): unknown => {
+ if (typeof value === "string") return value.replace(pattern, () => REDACTED_EVENT_VALUE);
+ if (Array.isArray(value)) return value.map(redact);
+ if (value && typeof value === "object") {
+ return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redact(entry)]));
+ }
+ return value;
+ };
+ return redact(input) as T;
+}
+
+/** Snapshot only diagnostics. Never walk a request, response, config or prompt. */
+export function collectRunFailureDiagnostics(run: Run, options: RunFailureReportOptions): RunFailureDiagnostics {
+ const execution = scalars(run, [
+ "runtimeMode", "executionStage", "nativePhase", "driverKind", "driverVersion",
+ ]);
+ if (options.phase) execution.failurePhase = options.phase;
+ if (run.startedAt && run.finishedAt) {
+ const durationMs = run.finishedAt.getTime() - run.startedAt.getTime();
+ if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;
+ }
+ const result = run.resultJson;
+ Object.assign(execution, scalars(result, [
+ "mode", "stopReason", "timeoutFired", "timeoutSource", "timeoutConfigured",
+ "effectiveTimeoutSec", "errorFamily",
+ ]));
+ const adapter = scalars(options.adapterErrorMeta, [
+ "category", "phase", "errorName", "acpCode", "causeMessage", "retryable",
+ "stackPreview", "status", "statusCode", "requestId",
+ ]);
+ const provider = scalars(read(result, "terminalSessionFailure"), ["category", "title", "details"]);
+ const truncatedFields: string[] = [];
+ const providerTruncation = read(read(result, "terminalSessionFailure"), "truncatedFields");
+ if (Array.isArray(providerTruncation)) {
+ for (const field of ["title", "details"]) {
+ if (providerTruncation.includes(field)) truncatedFields.push(`provider.${field}`);
+ }
+ }
+ const exceptions: RunFailureException[] = [];
+ const seen = new Set<unknown>();
+ let error = options.error;
+ while (error !== undefined && error !== null && exceptions.length < 4) {
+ if (seen.has(error)) {
+ truncatedFields.push("exceptions.cycle");
+ break;
+ }
+ seen.add(error);
+ if (typeof error === "string") {
+ exceptions.push({ message: error });
+ error = undefined;
+ break;
+ }
+ if (typeof error !== "object") break;
+ const entry: RunFailureException = {};
+ for (const field of ["name", "message", "stack", "code"] as const) {
+ const value = read(error, field);
+ if (typeof value === "string") entry[field] = value;
+ else if (field === "code" && typeof value === "number" && Number.isFinite(value)) entry.code = String(value);
+ }
+ const status = read(error, "status") ?? read(error, "statusCode");
+ if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) entry.status = status;
+ const requestId = read(error, "requestId") ?? read(error, "request_id");
+ if (typeof requestId === "string") entry.requestId = requestId;
+ if (Object.keys(entry).length > 0) exceptions.push(entry);
+ else break;
+ error = read(error, "cause");
+ }
+ if (error !== undefined && error !== null && exceptions.length === 4) truncatedFields.push("exceptions.depth");
+ return { execution, adapter, provider, exceptions, truncatedFields };
+}
+
+/** Redact complete values before cutting, including credentials across a cut. */
+export function sanitizeRunFailureText(input: string, limit: number): string {
+ const normalized = input
+ .replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
+ .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "")
+ .replace(/[\ud800-\udfff]/gu, "\ufffd");
+ const clean = redactSensitiveText(redactDiagnosticText(redactCurrentUserText(normalized)));
+ if (clean.length <= limit) return clean;
+ const suffix = "\n[truncated]";
+ let end = limit - suffix.length;
+ if ((clean.codePointAt(end - 1) ?? 0) > 0xffff) end--;
+ return clean.slice(0, end) + suffix;
+}
+
+/** Called after the run's registered secret values have also been removed. */
+export function sanitizeRunFailureDiagnostics(raw: RunFailureDiagnostics): RunFailureDiagnostics {
+ const truncatedFields = [...raw.truncatedFields];
+ const text = (value: string, path: string, limit: number) => {
+ const sanitized = sanitizeRunFailureText(value, limit);
+ if (sanitized.endsWith("\n[truncated]")) truncatedFields.push(path);
+ return sanitized;
+ };
+ const context = (value: Context, prefix: string, limits: Record<string, number> = {}): Context =>
+ Object.fromEntries(Object.entries(value).map(([key, value]) => [key,
+ typeof value === "string" ? text(value, `${prefix}.${key}`, limits[key] ?? 200) : value,
+ ]));
+ return {
+ execution: context(raw.execution, "execution"),
+ adapter: context(raw.adapter, "adapter", { causeMessage: 2048, stackPreview: 8192 }),
+ provider: context(raw.provider, "provider", { title: 4096, details: 12288 }),
+ exceptions: raw.exceptions.map((entry, i) => context(entry as Context, `exceptions.${i}`, { message: 2048, stack: 8192 })),
+ truncatedFields: [...new Set(truncatedFields)],
+ };
+}
diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts
index 7e40345513..0c044b6615 100644
--- a/server/src/services/run-failure-report.ts
+++ b/server/src/services/run-failure-report.ts
@@ -1,8 +1,14 @@
-import { eq } from "drizzle-orm";
+import { and, eq } from "drizzle-orm";
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
import { captureRunFailure, type RunFailureStatus } from "../sentry.js";
-import { redactCurrentUserText } from "../log-redaction.js";
-import { redactSensitiveText } from "../redaction.js";
+import {
+ collectRunFailureDiagnostics,
+ collectRunFailureSecretValues,
+ redactRunFailureSecretValues,
+ sanitizeRunFailureDiagnostics,
+ sanitizeRunFailureText,
+ type RunFailureReportOptions,
+} from "./run-failure-diagnostics.js";
import { logger } from "../middleware/logger.js";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
@@ -23,15 +29,6 @@ const pendingRunFailureReports = new Set<Promise<void>>();
/** Bounds the shutdown wait, so one stuck report cannot hang the process exit. */
const PENDING_REPORT_DRAIN_TIMEOUT_MS = 5_000;
-/**
- * Remove a credential and the current user's home path from adapter-supplied
- * text, then cut the result to `maxLength`. Run the length cut after the
- * redaction, so a credential cannot survive at a cut boundary.
- */
-function sanitizeAdapterText(input: string, maxLength: number): string {
- return redactSensitiveText(redactCurrentUserText(input)).slice(0, maxLength);
-}
-
function isRunFailureStatus(status: string): status is RunFailureStatus {
return status === "failed" || status === "timed_out";
}
@@ -53,10 +50,10 @@ function readTaskId(run: HeartbeatRun): string | null {
* its own in-flight promise, so a caller that does not await it still lets
* shutdown find and wait for the report β€” see `waitForPendingRunFailureReports`.
*/
-export function reportRunFailure(db: Db, run: HeartbeatRun): Promise<void> {
+export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureReportOptions = {}): Promise<void> {
if (!isRunFailureStatus(run.status)) return Promise.resolve();
const runStatus = run.status;
- const report = captureTerminalRunFailure(db, run, runStatus);
+ const report = captureTerminalRunFailure(db, run, runStatus, options);
pendingRunFailureReports.add(report);
void report.finally(() => pendingRunFailureReports.delete(report));
return report;
@@ -66,12 +63,21 @@ async function captureTerminalRunFailure(
db: Db,
run: HeartbeatRun,
runStatus: RunFailureStatus,
+ options: RunFailureReportOptions,
): Promise<void> {
try {
+ const snapshot = redactRunFailureSecretValues({
+ errorMessage: run.error ?? "",
+ errorCode: run.errorCode ?? null,
+ diagnostics: collectRunFailureDiagnostics(run, options),
+ }, [...new Set([
+ ...collectRunFailureSecretValues(process.env, [], true),
+ ...(options.secretValues ?? []),
+ ])].sort((a, b) => b.length - a.length));
const agent = await db
.select({ adapterType: agents.adapterType })
.from(agents)
- .where(eq(agents.id, run.agentId))
+ .where(and(eq(agents.id, run.agentId), eq(agents.companyId, run.companyId)))
.then((rows) => rows[0] ?? null);
const taskId = readTaskId(run);
@@ -80,18 +86,26 @@ async function captureTerminalRunFailure(
return;
}
+ // Resolve registered values before truncation. A failed resolution must
+ // not send an incompletely redacted report.
+ let redacted = snapshot;
+ if (Array.isArray(run.contextSnapshot?.paperclipSecretRedactions)) {
+ const { createRunSecretRedactionRegistry } = await import("./run-secret-redaction.js");
+ redacted = await createRunSecretRedactionRegistry(db).redactForRun(run.companyId, run.id, snapshot);
+ }
captureRunFailure({
taskId,
runId: run.id,
- errorMessage: sanitizeAdapterText(run.error ?? "", MAX_ERROR_MESSAGE_LENGTH),
+ errorMessage: sanitizeRunFailureText(redacted.errorMessage, MAX_ERROR_MESSAGE_LENGTH),
errorCode:
- run.errorCode === null
+ redacted.errorCode === null
? null
- : sanitizeAdapterText(run.errorCode, MAX_ERROR_CODE_LENGTH),
+ : sanitizeRunFailureText(redacted.errorCode, MAX_ERROR_CODE_LENGTH),
agentAdapter: agent?.adapterType ?? UNKNOWN_ADAPTER,
runStatus,
exitCode: run.exitCode,
signal: run.signal,
+ diagnostics: sanitizeRunFailureDiagnostics(redacted.diagnostics),
});
} catch (err) {
logger.warn({ err, runId: run.id }, "failed to report run failure to Sentry");