The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(sentry): preserve run failure stacks and diagnostic context (#14585)
Dottab3eb03fcbaaa
diff --git a/doc/observability.md b/doc/observability.mdindex 8a379342a1..93513a8071 100644--- a/doc/observability.md+++ b/doc/observability.md@@ -415,6 +415,37 @@ values become `unknown`. A signal such as `SIGKILL` does not establish who sentit or prove an out-of-memory kill. These fields do not change error groupingor run outcomes, and do not include process output or adapter result payloads.+The shared reporter also attaches bounded diagnostic contexts for both legacy+and native runs:++- `run_execution`: runtime mode, execution stage/native phase, driver and version,+ duration, failure phase, stop reason, error family, and timeout settings when available.+- `adapter_failure`: selected adapter error fields such as phase, category,+ protocol code, retryability, cause message, stack preview, HTTP status, and request ID.+- `provider_failure`: the saved provider failure category, title, and details.+- `run_exception_0` through `run_exception_3`: exception names, codes, HTTP+ statuses, and request IDs for a caught exception and up to three causes.++The execution and setup catch paths pass the original exception to the reporter.+It snapshots and rebuilds only these selected fields and the original message and+stack. Sentry receives the sanitized cause chain rather than a stack created at+the reporting call. Saved adapter results use an available adapter stack preview;+without one, the report has no synthetic reporting stack.++Credential patterns, the current user's home path, registered run-secret values,+and known host/runtime environment credentials are removed before truncation.+Declared environment secret bindings are included even when their key has no+credential-like name. Runtime values are used only for redaction and are never+copied into an event. Stack fields are limited to 8,192 characters,+exception messages to 2,048, provider titles to 4,096, and provider details to+12,288. Other diagnostic strings are limited to 200 characters (adapter cause+messages: 2,048). Truncation is marked in the text and in+`run_execution.truncatedFields`; cyclic and deeper cause chains are marked too.+Request/response objects, headers, environment/configuration, prompts, stdout,+stderr, and arbitrary adapter result fields are not copied. The existing DSN+opt-in gate and error-code/adapter fingerprint remain unchanged. These reports+cannot recover diagnostic data that a provider or adapter discarded upstream.+### Browser dataThe browser sends no page URL, no referrer, no user agent, and nodiff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.tsindex 39d9e150fe..6798347819 100644--- a/server/src/__tests__/chat-channels.integration.test.ts+++ b/server/src/__tests__/chat-channels.integration.test.ts@@ -1048,7 +1048,19 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {try {await Promise.all([...fixtureServices].map((service) => service.shutdown()));} finally {- await retireFixtureState([...fixtureCompanies]);+ const companyIds = [...fixtureCompanies];+ await retireFixtureState(companyIds);+ if (companyIds.length > 0) {+ // Pausing an endpoint does not remove its rows from global recovery+ // selectors. After every assertion and worker shutdown, settle leftover+ // fixture work so later cases cannot claim its leases or retry its I/O.+ await db.update(chatActions).set({ status: "cancelled" })+ .where(and(inArray(chatActions.companyId, companyIds), notInArray(chatActions.status, ["processed", "cancelled"])));+ await db.update(chatDeliveries).set({ state: "failed", nextAttemptAt: null })+ .where(and(inArray(chatDeliveries.companyId, companyIds), inArray(chatDeliveries.state, ["received", "processing", "retry"])));+ await db.update(chatPublications).set({ state: "cancelled", nextAttemptAt: null })+ .where(and(inArray(chatPublications.companyId, companyIds), inArray(chatPublications.state, ["pending", "awaiting_consent", "streaming", "retry"])));+ }fixtureServices.clear();fixtureCompanies.clear();}diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex 3d95884685..f6d391e0d5 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -1565,8 +1565,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {"continuation_task_ownership_changed",])("retains untyped continuation setup failures: %s", async (message) => {const { runId, wakeupRequestId } = await seedQueuedIssueRunFixture();+ const error = new Error(message, { cause: Object.assign(new Error("upstream setup failed"), { code: "ECONNRESET" }) });const build = vi.spyOn(executionContinuation, "buildExecutionContinuation")- .mockRejectedValueOnce(new Error(message));+ .mockRejectedValueOnce(error);try {const heartbeat = heartbeatService(db);await heartbeat.resumeQueuedRuns();@@ -1576,6 +1577,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {expect(await heartbeat.getRun(runId)).toMatchObject({status: "failed", errorCode: "setup_failed", error: message,});+ const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]);+ expect(report?.diagnostics).toMatchObject({+ execution: { failurePhase: "setup" },+ exceptions: [{ message, stack: expect.stringContaining("heartbeat-process-recovery.test.ts") }, { code: "ECONNRESET" }],+ });const [wakeup] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId));expect(wakeup.status).toBe("failed");@@ -4801,6 +4807,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {expect(failedRun?.error).toContain("is not installed or its plugin worker is not running",);+ const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]);+ expect(report?.diagnostics).toMatchObject({+ execution: { failurePhase: "execute" },+ exceptions: [{ stack: expect.stringContaining("heartbeat-process-recovery.test.ts") }],+ });const interaction = await waitForValue(async () => {const row = await db@@ -5282,6 +5293,38 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {expect(validationComment).toBeTruthy();});+ it.each(["throw", "result"])("redacts opaque environment-bound credentials from Sentry diagnostics: %s", async (mode) => {+ const { companyId, agentId, runId } = await seedQueuedIssueRunFixture();+ const svc = secretService(db);+ const value = "opaque-runtime-credential-fixture";+ const secret = await svc.create(companyId, {+ name: `sentry-redaction-${randomUUID()}`, provider: "local_encrypted", value,+ });+ const env = { CUSTOM_BINDING: { type: "secret_ref", secretId: secret.id, version: "latest" } };+ await svc.syncEnvBindingsForTarget(companyId, { targetType: "agent", targetId: agentId }, env);+ await db.update(agents).set({ adapterConfig: { env } }).where(eq(agents.id, agentId));+ mockAdapterExecute.mockImplementationOnce(async (input) => {+ expect(input.config.env.CUSTOM_BINDING).toBe(value);+ const message = `upstream rejected ${value}`;+ if (mode === "throw") throw new Error(message, { cause: new Error(message) });+ return {+ exitCode: 1, signal: null, timedOut: false, errorMessage: message, errorCode: "adapter_failed",+ errorMeta: { causeMessage: message },+ resultJson: { terminalSessionFailure: { category: "service", details: message } },+ };+ });+ const heartbeat = heartbeatService(db);+ await heartbeat.resumeQueuedRuns();+ await waitForRunToSettle(heartbeat, runId, 5_000);+ await heartbeat.waitForRunExecutionDrain(runId);+ const report = await waitForValue(async () => mockCaptureRunFailure.mock.calls.find(([event]) => event.runId === runId)?.[0]);+ expect(report).toBeDefined();+ expect(report.errorMessage).toContain("upstream rejected");+ expect(JSON.stringify(report)).not.toContain(value);+ if (mode === "throw") expect(report.diagnostics.exceptions).toHaveLength(2);+ else expect(report.diagnostics.provider.category).toBe("service");+ });+it("blocks before dispatch when a declared secret ref has no binding instead of emitting an opaque setup failure", async () => {const { companyId, agentId, runId, issueId } =await seedQueuedIssueRunFixture();diff --git a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts b/server/src/__tests__/run-failure-sentry-real-sdk.test.tsindex 5cfe758e2b..11b3c259f5 100644--- a/server/src/__tests__/run-failure-sentry-real-sdk.test.ts+++ b/server/src/__tests__/run-failure-sentry-real-sdk.test.ts@@ -1,5 +1,7 @@import { createRequire } from "node:module";import { afterEach, describe, expect, it, vi } from "vitest";+import { collectRunFailureDiagnostics, sanitizeRunFailureDiagnostics } from "../services/run-failure-diagnostics.js";+import type { heartbeatRuns } from "@paperclipai/db";// Sentry is an optional peer. When installed, exercise the real SDK with an// in-memory transport, including its context behavior without an OTel manager.@@ -76,7 +78,17 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK",exitCode: null,signal: "SIGKILL",};- captureRunFailure(first);+ const original = Object.assign(new Error(first.errorMessage, { cause: Object.assign(new Error("upstream connection reset"), {+ code: "ECONNRESET", requestId: "request-123", status: 503,+ stack: "Error: upstream connection reset\n at socketRead (/app/provider.js:19:7)",+ response: { body: "private-response" },+ }) }), { stack: "Error: first run failed\n at originalAdapter (/app/adapter.js:42:7)", request: { headers: "private-headers" } });+ const diagnostics = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics({+ runtimeMode: "legacy", resultJson: { terminalSessionFailure: {+ category: "service", title: "Provider failed", details: "d".repeat(9000),+ } },+ } as unknown as typeof heartbeatRuns.$inferSelect, { error: original, phase: "execute" }));+ captureRunFailure({ ...first, diagnostics });await Promise.resolve();captureException(new Error("unrelated database error"));captureRunFailure(second);@@ -91,7 +103,7 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK",expect(events).toHaveLength(5);const captured = (message: string) => events.find((event) =>- (event.exception as { values: Array<{ value: string }> }).values[0]?.value === message,+ (event.exception as { values: Array<{ value: string }> }).values.some((value) => value.value === message),);for (const run of [first, second]) {expect(captured(run.errorMessage)).toMatchObject({@@ -108,10 +120,25 @@ describe.skipIf(!sentryPackage)("run failure context with the real Sentry SDK",fingerprint: [first.errorCode, first.agentAdapter],});expect(JSON.stringify(events)).not.toContain("private-signal-payload");+ const firstCapture = captured(first.errorMessage)!;+ const exceptions = (firstCapture.exception as { values: Array<Record<string, unknown>> }).values;+ expect(exceptions).toHaveLength(2);+ expect(JSON.stringify(exceptions)).toContain("originalAdapter");+ expect(JSON.stringify(exceptions)).toContain("socketRead");+ expect(JSON.stringify(exceptions)).not.toContain("captureRunFailure");+ expect(JSON.stringify(captured(second.errorMessage)?.exception)).not.toContain("captureRunFailure");+ expect(firstCapture).toMatchObject({ contexts: {+ provider_failure: { category: "service", details: "d".repeat(9000) },+ run_exception_1: { code: "ECONNRESET", requestId: "request-123", status: 503 },+ } });+ expect(JSON.stringify(events)).not.toContain("private-");for (const message of ["unrelated database error", "unrelated filesystem error"]) {const event = captured(message);expect(event).toBeDefined();expect(event).not.toHaveProperty("contexts.run_failure");+ for (const context of ["run_execution", "adapter_failure", "provider_failure", "run_exception_0", "run_exception_1"]) {+ expect(event).not.toHaveProperty(`contexts.${context}`);+ }expect(event).not.toHaveProperty("fingerprint");for (const tag of ["run_id", "task_id", "error_code", "agent_adapter", "run_status"]) {expect(event).not.toHaveProperty(`tags.${tag}`);diff --git a/server/src/__tests__/run-failure-sentry.test.ts b/server/src/__tests__/run-failure-sentry.test.tsindex d5285698a4..9ba4c7467f 100644--- a/server/src/__tests__/run-failure-sentry.test.ts+++ b/server/src/__tests__/run-failure-sentry.test.ts@@ -215,6 +215,31 @@ describe("captureRunFailure", () => {expect((received as Error).message).toBe("boom");});+ it("uses sanitized original stacks and causes instead of the reporting stack", async () => {+ const { sentryModule, captureException } = await importFreshSentryWithGateOpen();+ sentryModule.captureRunFailure(baseEvent({ diagnostics: {+ execution: { runtimeMode: "native", failurePhase: "setup" },+ adapter: {}, provider: {}, truncatedFields: [],+ exceptions: [+ { name: "TypeError", message: "setup failed", stack: "TypeError: setup failed\n at originalSetup (/app/setup.js:42:7)" },+ { name: "Error", message: "connection reset", stack: "Error: connection reset\n at socketRead (/app/network.js:9:4)", code: "ECONNRESET", requestId: "req-123" },+ ],+ } }));+ const [error, context] = captureException.mock.calls[0]!;+ expect(error).toBeInstanceOf(Error);+ expect(error).toMatchObject({ name: "TypeError", message: "setup failed", cause: { message: "connection reset" } });+ expect((error as Error).stack).toContain("originalSetup");+ expect((error as Error).stack).not.toContain("captureRunFailure");+ expect(context?.contexts.run_exception_1).toMatchObject({ code: "ECONNRESET", requestId: "req-123" });+ expect(context?.contexts.run_execution).toMatchObject({ failurePhase: "setup" });+ });++ it("does not invent a reporter stack for a saved result with no original exception", async () => {+ const { sentryModule, captureException } = await importFreshSentryWithGateOpen();+ sentryModule.captureRunFailure(baseEvent());+ expect((captureException.mock.calls[0]![0] as Error).stack).toBeUndefined();+ });+it("does not throw and captures nothing when the gate is closed", async () => {vi.resetModules();const sentryModule = await import("../sentry.js");diff --git a/server/src/sentry.ts b/server/src/sentry.tsindex 1a9c30f464..ee7f28f49d 100644--- a/server/src/sentry.ts+++ b/server/src/sentry.ts@@ -56,6 +56,7 @@// `instrumentation.ts`.import os from "node:os";+import type { RunFailureDiagnostics } from "./services/run-failure-diagnostics.js";import { readBuildCommit } from "./build-commit.js";import { checkExactPeerVersions } from "./peer-version-check.js";import { resolveSentryDsns } from "./sentry-dsn.js";@@ -132,6 +133,8 @@ export interface RunFailureEvent {agentAdapter: string;/** The run status that triggered this report. */runStatus: RunFailureStatus;+ /** Bounded, redacted diagnostics selected by the run failure reporter. */+ diagnostics?: RunFailureDiagnostics;/** Recorded process exit evidence, when available. Validated before capture. */exitCode?: number | null;signal?: string | null;@@ -174,7 +177,33 @@ export function captureRunFailure(event: RunFailureEvent): void {// Sentry's async scope isolation is absent when OTel setup is skipped.// A withScope mutation can then persist into unrelated later captures.// Pass these fields on this event only; do not mutate the ambient scope.- handle.captureException(new Error(event.errorMessage), {+ const diagnostics = event.diagnostics;+ const contexts: Record<string, Record<string, unknown>> = {};+ if (diagnostics) {+ contexts.run_execution = { ...diagnostics.execution, truncatedFields: diagnostics.truncatedFields };+ if (Object.keys(diagnostics.adapter).length) contexts.adapter_failure = diagnostics.adapter;+ if (Object.keys(diagnostics.provider).length) contexts.provider_failure = diagnostics.provider;+ diagnostics.exceptions.forEach(({ name, code, status, requestId }, index) => {+ contexts[`run_exception_${index}`] = { name, code, status, requestId };+ });+ }+ // Rebuild only sanitized exception fields. Passing a raw SDK Error can+ // serialize its request/response, headers, or other enumerable properties.+ let cause: Error | undefined;+ for (const entry of [...(diagnostics?.exceptions ?? [])].reverse()) {+ const error: Error = new Error(entry.message ?? event.errorMessage, cause ? { cause } : undefined);+ error.name = entry.name ?? "Error";+ error.stack = entry.stack;+ cause = error;+ }+ const exception = cause ?? new Error(event.errorMessage);+ if (!cause) {+ // A saved adapter result is not a thrown Error. Do not pretend that the+ // reporter's own stack is the failure location.+ exception.stack = typeof diagnostics?.adapter.stackPreview === "string"+ ? diagnostics.adapter.stackPreview : undefined;+ }+ handle.captureException(exception, {tags: {run_id: event.runId,task_id: event.taskId,@@ -183,6 +212,7 @@ export function captureRunFailure(event: RunFailureEvent): void {run_status: event.runStatus,},contexts: {+ ...contexts,run_failure: {taskId: event.taskId,runId: event.runId,diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.tsnew file mode 100644index 0000000000..838d63f8d0--- /dev/null+++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts@@ -0,0 +1,95 @@+import { describe, expect, it } from "vitest";+import type { heartbeatRuns } from "@paperclipai/db";+import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js";++type Run = typeof heartbeatRuns.$inferSelect;+const run = (overrides: Partial<Run> = {}) => ({ resultJson: null, ...overrides }) as Run;+const collect = (error: unknown) => collectRunFailureDiagnostics(run(), { error });++describe("run failure diagnostics", () => {+ it("selects declared environment secrets under opaque names and common credential keys", () => {+ expect(collectRunFailureSecretValues({+ CUSTOM_BINDING: "bound-opaque-value", ACCESS_TOKEN: "plain-opaque-value", REGION: "us-east-1", EMPTY_KEY: "",+ }, ["CUSTOM_BINDING"])).toEqual(["bound-opaque-value", "plain-opaque-value", "us-east-1"]);+ expect(collectRunFailureSecretValues({ PATH: "/usr/bin", CUSTOM_BINDING: "opaque" }, [], true)).toEqual(["opaque"]);+ expect(collectRunFailureSecretValues({ PATH: "secret-override" }, ["PATH"], true)).toEqual(["secret-override"]);+ });++ it("redacts encoded credentials and URL passwords without expanding replacement markers", () => {+ const secret = 'opaque/"credential';+ const values = collectRunFailureSecretValues({ DATABASE_URL: `postgres://user:${encodeURIComponent(secret)}@example/db` });+ const result = redactRunFailureSecretValues({ text: `${secret} ${encodeURIComponent(secret)} ${JSON.stringify(secret)}`, status: 503 }, [...values, "REDACTED"]);+ expect(result.text).not.toContain("opaque");+ expect(result.status).toBe(503);+ expect(result.text).not.toContain("***[REDACTED]");+ });++ it("preserves known boolean and numeric settings without exempting unknown or declared secrets", () => {+ const env = { PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_DB_BACKUP_RETENTION_DAYS: "1" };+ expect(collectRunFailureSecretValues(env, [], true)).toEqual([]);+ expect(collectRunFailureSecretValues(env, ["PAPERCLIP_DB_BACKUP_ENABLED"], true)).toEqual(["false"]);+ expect(collectRunFailureSecretValues({ CUSTOM_BINDING: "1" }, [], true)).toEqual(["1"]);+ expect(collectRunFailureSecretValues({ PAPERCLIP_DB_BACKUP_ENABLED: "opaque" }, [], true)).toEqual(["opaque"]);+ });+ it("preserves generic exceptions, numeric codes and nested network causes", () => {+ const root = Object.assign(new Error("network failed"), { code: "ECONNRESET", statusCode: 502, request_id: "req-123" });+ const outer = Object.assign(new TypeError("adapter failed", { cause: root }), { code: -32000 });+ const result = sanitizeRunFailureDiagnostics(collect(outer));+ expect(result.exceptions).toMatchObject([+ { name: "TypeError", message: "adapter failed", code: "-32000" },+ { name: "Error", message: "network failed", code: "ECONNRESET", status: 502, requestId: "req-123" },+ ]);+ expect(result.exceptions[0].stack).toContain("run-failure-diagnostics.test.ts");+ });++ it("bounds cyclic, deep, and string causes without inspecting arbitrary error objects", () => {+ const cyclic = new Error("cycle"); cyclic.cause = cyclic;+ expect(collect(cyclic)).toMatchObject({ exceptions: [{ message: "cycle" }], truncatedFields: ["exceptions.cycle"] });+ let error = new Error("leaf");+ for (let i = 0; i < 10; i++) error = new Error(`level-${i}`, { cause: error });+ expect(collect(error).exceptions).toHaveLength(4);+ expect(collect(error).truncatedFields).toContain("exceptions.depth");+ expect(collect(new Error("outer", { cause: "string cause" })).exceptions).toHaveLength(2);+ const getter = Object.defineProperty({ message: "valid message" }, "response", { get: () => { throw new Error("must not read"); } });+ expect(collect(getter).exceptions).toEqual([{ message: "valid message" }]);+ expect(collect({ arbitrary: "private payload" }).exceptions).toEqual([]);+ expect(collect(Object.defineProperty({}, "message", { get: () => { throw new Error("bad getter"); } })).exceptions).toEqual([]);+ });++ it("redacts credential forms before truncation in all diagnostic text", () => {+ const failure = collectRunFailureDiagnostics(run({ resultJson: {+ terminalSessionFailure: { category: "service", title: 'password="secret-password"', details: 'Authorization: Bearer secret-bearer' },+ } }), { error: Object.assign(new Error('apiKey="secret-api-key"'), { cause: new Error('token="secret-token"') }), adapterErrorMeta: { causeMessage: 'password="secret-password"' } });+ const result = sanitizeRunFailureDiagnostics(failure);+ expect(JSON.stringify(result)).not.toContain("secret-");+ expect(sanitizeRunFailureText('x'.repeat(90) + ' password="secret-at-cut-boundary"', 110)).not.toContain("secret-at");+ expect(sanitizeRunFailureText('pass\x1b[31mword="secret-colored"', 200)).not.toContain("secret-colored");+ });++ it("reports truncation, preserves UTF-16 pairs, and keeps the total payload bounded", () => {+ const huge = "π".repeat(100_000);+ let error = Object.assign(new Error(huge), { stack: huge });+ for (let i = 0; i < 5; i++) error = Object.assign(new Error(huge, { cause: error }), { stack: huge });+ const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ resultJson: {+ terminalSessionFailure: { category: "service", title: huge, details: huge, truncatedFields: ["details", "private-field"] },+ } }), { error }));+ expect(String(result.provider.title).length).toBeLessThanOrEqual(4096);+ expect(String(result.provider.details).length).toBeLessThanOrEqual(12288);+ expect(JSON.stringify(result)).not.toMatch(/[\ud800-\udfff]/u);+ expect(result.truncatedFields).toContain("provider.details");+ expect(result.truncatedFields).toContain("exceptions.0.stack");+ expect(result.truncatedFields).not.toContain("provider.private-field");+ expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan(150_000);+ });++ it("keeps scalar native and legacy execution evidence and rejects non-finite durations", () => {+ const result = collectRunFailureDiagnostics(run({+ runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3",+ startedAt: new Date(1000), finishedAt: new Date(8500),+ resultJson: { stopReason: "failed", timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream", summary: "private prose" },+ }), { phase: "execute" });+ expect(result.execution).toMatchObject({ runtimeMode: "native", nativePhase: "turn", driverKind: "codex", driverVersion: "1.2.3", durationMs: 7500, timeoutFired: true, effectiveTimeoutSec: 60, errorFamily: "transient_upstream" });+ expect(JSON.stringify(result)).not.toContain("private prose");+ expect(collectRunFailureDiagnostics(run({ startedAt: new Date(NaN), finishedAt: new Date() }), {}).execution).not.toHaveProperty("durationMs");+ });+});diff --git a/server/src/services/__tests__/run-failure-report.test.ts b/server/src/services/__tests__/run-failure-report.test.tsindex 9b32245e76..2b7e184e48 100644--- a/server/src/services/__tests__/run-failure-report.test.ts+++ b/server/src/services/__tests__/run-failure-report.test.ts@@ -1,6 +1,6 @@import { randomUUID } from "node:crypto";import os from "node:os";-import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";import { agents, companies, createDb, heartbeatRuns, type Db } from "@paperclipai/db";import {getEmbeddedPostgresTestSupport,@@ -9,6 +9,11 @@ import {const mockCaptureRunFailure = vi.hoisted(() => vi.fn());const mockRedactCurrentUserText = vi.hoisted(() => vi.fn());+const mockResolveSecret = vi.hoisted(() => vi.fn());++vi.mock("../../secrets/provider-registry.js", () => ({+ getSecretProvider: () => ({ resolveVersion: mockResolveSecret }),+}));vi.mock("../../sentry.js", () => ({captureRunFailure: mockCaptureRunFailure,@@ -23,8 +28,7 @@ vi.mock("../../log-redaction.js", async (importOriginal) => {});import { reportRunFailure, waitForPendingRunFailureReports } from "../run-failure-report.js";-import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../../redaction.js";-import { redactCurrentUserText } from "../../log-redaction.js";+import { REDACTED_EVENT_VALUE } from "../../redaction.js";const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;@@ -34,14 +38,29 @@ describeEmbeddedPostgres("reportRunFailure", () => {let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;let companyId!: string;let agentId!: string;+ let inheritedEnv: NodeJS.ProcessEnv;beforeAll(async () => {tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-failure-report-");db = createDb(tempDb.connectionString);}, 20_000);+ beforeEach(() => {+ // Unknown host values intentionally count as secrets, including short values+ // such as "1". Keep these fixtures independent of the developer/CI environment+ // and add secret values explicitly in the tests that exercise redaction.+ inheritedEnv = process.env;+ process.env = Object.fromEntries(+ ["PATH", "HOME", "USER", "USERNAME", "LOGNAME", "USERPROFILE", "TMPDIR", "TEMP", "TMP"]+ .flatMap((key) => inheritedEnv[key] === undefined ? [] : [[key, inheritedEnv[key]]]),+ );+ });+afterEach(async () => {vi.clearAllMocks();+ vi.unstubAllEnvs();+ process.env = inheritedEnv;+ await db.delete(heartbeatRuns);await db.delete(agents);await db.delete(companies);});@@ -141,6 +160,98 @@ describeEmbeddedPostgres("reportRunFailure", () => {}});+ it("reports selected provider diagnostics and the original cause without copying arbitrary payloads", async () => {+ await seedCompanyAndAgent();+ const cause = Object.assign(new Error("provider unreachable"), {+ code: "ECONNRESET", requestId: "request-123", status: 503,+ response: { body: "private-response" },+ });+ const error = new Error("adapter threw", { cause });+ const run = buildRun({+ runtimeMode: "legacy",+ executionStage: "execute",+ error: "adapter threw",+ stderrExcerpt: "private-stderr",+ stdoutExcerpt: "private-stdout",+ contextSnapshot: { prompt: "private-prompt" },+ resultJson: {+ terminalSessionFailure: { category: "service", title: "Provider unavailable", details: "request-123 failed", raw: "private-provider-raw" },+ timeoutFired: false, summary: "private-summary", env: { SECRET: "private-env" },+ },+ });+ await reportRunFailure(db, run, { error, phase: "execute", adapterErrorMeta: { phase: "turn", retryable: true, response: "private-adapter-response" } });+ const captured = mockCaptureRunFailure.mock.calls[0][0];+ expect(captured.diagnostics).toMatchObject({+ execution: { runtimeMode: "legacy", executionStage: "execute", failurePhase: "execute", timeoutFired: false },+ adapter: { phase: "turn", retryable: true },+ provider: { category: "service", title: "Provider unavailable", details: "request-123 failed" },+ exceptions: [{ message: "adapter threw" }, { message: "provider unreachable", code: "ECONNRESET", status: 503, requestId: "request-123" }],+ });+ expect(captured.diagnostics.exceptions[0].stack).toContain("run-failure-report.test.ts");+ expect(JSON.stringify(captured)).not.toContain("private-");+ expect(error.cause).toBe(cause);+ });++ it.each([false, true])("redacts registered run secrets and fails closed when resolution fails: %s", async (fails) => {+ await seedCompanyAndAgent();+ const secret = "opaque-registered-value";+ const run = buildRun({+ error: `connection failed: ${secret}`,+ contextSnapshot: { paperclipSecretRedactions: [{ fingerprintSha256: "fixture", material: { encrypted: "fixture" } }] },+ resultJson: { terminalSessionFailure: { category: "service", details: `upstream rejected ${secret}` } },+ });+ await db.insert(heartbeatRuns).values(run);+ if (fails) mockResolveSecret.mockRejectedValueOnce(new Error("fixture resolution failed"));+ else mockResolveSecret.mockResolvedValueOnce(secret);++ await expect(reportRunFailure(db, run, {+ error: new Error(`failed ${secret}`, { cause: new Error(`cause ${secret}`) }),+ adapterErrorMeta: { causeMessage: `adapter ${secret}` },+ })).resolves.toBeUndefined();+ expect(mockResolveSecret).toHaveBeenCalledTimes(1);+ if (fails) {+ expect(mockCaptureRunFailure).not.toHaveBeenCalled();+ } else {+ expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);+ const captured = mockCaptureRunFailure.mock.calls[0][0];+ expect(JSON.stringify(captured)).not.toContain(secret);+ expect(captured.diagnostics.provider.details).toContain(REDACTED_EVENT_VALUE);+ expect(captured.diagnostics.exceptions[1].message).toContain(REDACTED_EVENT_VALUE);+ }+ });++ it("redacts runtime and host environment secret values even without a persisted registry", async () => {+ await seedCompanyAndAgent();+ const runtimeSecret = "opaque-runtime-value";+ const hostSecret = "opaque-host-value";+ vi.stubEnv("FIXTURE_CUSTOM_VALUE", hostSecret);+ const text = `connection failed: ${runtimeSecret} ${hostSecret}`;+ const run = buildRun({ error: text, contextSnapshot: null,+ resultJson: { terminalSessionFailure: { details: text } },+ });+ await reportRunFailure(db, run, {+ error: new Error(text, { cause: new Error(text) }),+ adapterErrorMeta: { causeMessage: text, stackPreview: text },+ secretValues: [runtimeSecret],+ });+ expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);+ const captured = mockCaptureRunFailure.mock.calls[0][0];+ expect(JSON.stringify(captured)).not.toContain(runtimeSecret);+ expect(JSON.stringify(captured)).not.toContain(hostSecret);+ expect(captured.diagnostics.exceptions).toHaveLength(2);+ expect(captured).not.toHaveProperty("secretValues");+ });++ it("still redacts short values from unknown host settings", async () => {+ await seedCompanyAndAgent();+ vi.stubEnv("FIXTURE_CUSTOM_VALUE", "1");++ await reportRunFailure(db, buildRun());++ expect(mockCaptureRunFailure.mock.calls[0][0].errorMessage)+ .toBe(`the provider process exited with code ${REDACTED_EVENT_VALUE}`);+ });+it("captures nothing for succeeded, cancelled, and interrupted", async () => {await seedCompanyAndAgent();for (const status of ["succeeded", "cancelled", "interrupted"] as const) {@@ -259,9 +370,10 @@ describeEmbeddedPostgres("reportRunFailure", () => {await reportRunFailure(db, run);- const expectedErrorMessage = redactSensitiveText(redactCurrentUserText(rawError));const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];- expect(errorMessage).toBe(expectedErrorMessage);+ // Environment-value redaction can fully mask the username before the+ // current-user redactor applies its partial mask. Both remove the home path.+ expect(errorMessage).toContain("workspace/report.log");expect(errorMessage).not.toContain(homeDir);});@@ -276,10 +388,13 @@ describeEmbeddedPostgres("reportRunFailure", () => {const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];expect(errorMessage).toHaveLength(MAX_ERROR_MESSAGE_LENGTH);- expect(errorMessage).toBe("x".repeat(MAX_ERROR_MESSAGE_LENGTH));+ expect(errorMessage).toBe("x".repeat(MAX_ERROR_MESSAGE_LENGTH - 12) + "\n[truncated]");});- it("does not change a short error message", async () => {+ it("preserves a short error message with known public host settings", async () => {+ vi.stubEnv("PAPERCLIP_DB_BACKUP_ENABLED", "false");+ vi.stubEnv("PAPERCLIP_DB_BACKUP_RETENTION_DAYS", "1");+ vi.stubEnv("GITHUB_RUN_ATTEMPT", "1");await seedCompanyAndAgent();const shortError = "the provider process exited with code 1";const run = buildRun({ status: "failed", error: shortError });@@ -326,7 +441,7 @@ describeEmbeddedPostgres("reportRunFailure", () => {const { errorCode } = mockCaptureRunFailure.mock.calls[0][0];expect(errorCode).toHaveLength(MAX_ERROR_CODE_LENGTH);- expect(errorCode).toBe("y".repeat(MAX_ERROR_CODE_LENGTH));+ expect(errorCode).toBe("y".repeat(MAX_ERROR_CODE_LENGTH - 12) + "\n[truncated]");});it("sends errorCode null unchanged when the run holds no error code", async () => {diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex 2f86e242fe..490101f2fd 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -326,6 +326,7 @@ import {emitAgentTaskRunById,} from "./agent-task-run-telemetry.js";import { reportRunFailure } from "./run-failure-report.js";+import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js";import { companySkillService } from "./company-skills.js";import { budgetService, type BudgetEnforcementScope } from "./budgets.js";import { secretService, type MissingRuntimeBinding } from "./secrets.js";@@ -12754,12 +12755,13 @@ export function heartbeatService(function emitTerminalAgentTaskRun(updated: typeof heartbeatRuns.$inferSelect,previousStatus: string | null,+ failureReport?: RunFailureReportOptions,) {if (!isHeartbeatRunTerminalStatus(updated.status)) return;if (previousStatus === updated.status) return;clearHeartbeatRunRuntimeStatus(updated.id);void emitAgentTaskRun(db, updated);- void reportRunFailure(db, updated);+ void reportRunFailure(db, updated, failureReport);}async function setRunStatus(@@ -12833,8 +12835,9 @@ export function heartbeatService(runId: string,status: string,patch?: Partial<typeof heartbeatRuns.$inferInsert>,+ failureReport?: RunFailureReportOptions,) {- return setRunStatusFromLive(runId, status, ["running"], patch);+ return setRunStatusFromLive(runId, status, ["running"], patch, failureReport);}// Move a run to a new status only when its current status is one of@@ -12847,6 +12850,7 @@ export function heartbeatService(status: string,fromStatuses: string[],patch?: Partial<typeof heartbeatRuns.$inferInsert>,+ failureReport?: RunFailureReportOptions,) {// fromStatuses can name a terminal status as its own source (for example,// an idempotent "still failed" patch), so the write below is not always a@@ -12916,7 +12920,7 @@ export function heartbeatService(payload: buildHeartbeatRunStatusLiveEventPayload(updated),});publishRunLifecyclePluginEvent(updated);- emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null);+ emitTerminalAgentTaskRun(updated, previousStatus?.status ?? null, failureReport);return { run: updated, updated: true as const };}@@ -20238,6 +20242,7 @@ export function heartbeatService(ReturnType<typeof traceStore.prepare>> | null = null;let providerTraceFinalized = false;+ let readFailureReportSecrets: () => string[] = () => [];try {const agent = await getAgent(run.agentId);@@ -21414,6 +21419,7 @@ export function heartbeatService(secretsSvc,trustPreset,});+ readFailureReportSecrets = () => collectRunFailureSecretValues(resolvedConfig.env, secretKeys);if (aiBinding) {try {managedAiRuntime = await prepareManagedAiRuntime(db, { companyId: agent.companyId, agentId: agent.id, responsibleUserId, adapterType: agent.adapterType, binding: aiBinding, config: resolvedConfig });@@ -21504,6 +21510,11 @@ export function heartbeatService(// Always replace this runtime-only field; caller wake data cannot supply skills.context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };let runtimeConfig: Record<string, unknown> = connectorDelivery.config;+ const resolvedFailureSecrets = readFailureReportSecrets();+ readFailureReportSecrets = () => [+ ...resolvedFailureSecrets,+ ...collectRunFailureSecretValues(runtimeConfig.env, secretKeys),+ ];const latestAgentConfigRevision = await getLatestAgentConfigRevision(agent.companyId,agent.id,@@ -25191,6 +25202,7 @@ export function heartbeatService(run.id,status,finalRunPatch,+ { adapterErrorMeta: adapterResult.errorMeta, secretValues: readFailureReportSecrets() },);let persistedRun: typeof heartbeatRuns.$inferSelect | null =persistedRunWrite.run;@@ -25830,7 +25842,7 @@ export function heartbeatService(logBytes: logSummary?.bytes,logSha256: logSummary?.sha256,logCompressed: logSummary?.compressed ?? false,- });+ }, { error: err, phase: "execute", secretValues: readFailureReportSecrets() });if (!failedRunWrite.updated &&!(@@ -26084,7 +26096,7 @@ export function heartbeatService(: setupFailureResultJson? { resultJson: setupFailureResultJson }: {}),- }).catch(() => ({ run: null, updated: false as const }));+ }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets() }).catch(() => ({ run: null, updated: false as const }));if (!setupFailureWrite.updated) {logger.info({diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.tsnew file mode 100644index 0000000000..86197fc1a1--- /dev/null+++ b/server/src/services/run-failure-diagnostics.ts@@ -0,0 +1,227 @@+import type { heartbeatRuns } from "@paperclipai/db";+import { redactDiagnosticText } from "@paperclipai/adapter-utils/command-redaction";+import { redactCurrentUserText } from "../log-redaction.js";+import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";++type Run = typeof heartbeatRuns.$inferSelect;+type Context = Record<string, string | number | boolean>;++export interface RunFailureReportOptions {+ /** The caught exception, before callers flatten it to a message. */+ error?: unknown;+ /** Structured adapter error metadata; only known diagnostic keys are read. */+ adapterErrorMeta?: unknown;+ phase?: "setup" | "execute";+ /** Runtime-only values to redact; never included in the captured event. */+ secretValues?: readonly string[];+}++export interface RunFailureException {+ name?: string;+ message?: string;+ stack?: string;+ code?: string;+ status?: number;+ requestId?: string;+}++export interface RunFailureDiagnostics {+ execution: Context;+ adapter: Context;+ provider: Context;+ exceptions: RunFailureException[];+ truncatedFields: string[];+}++function read(value: unknown, key: string): unknown {+ if (!value || typeof value !== "object") return undefined;+ try {+ return (value as Record<string, unknown>)[key];+ } catch {+ return undefined;+ }+}++function scalars(value: unknown, fields: readonly string[]): Context {+ const result: Context = {};+ for (const field of fields) {+ const entry = read(value, field);+ if (+ typeof entry === "string" || typeof entry === "boolean" ||+ (typeof entry === "number" && Number.isFinite(entry))+ ) result[field] = entry;+ }+ return result;+}++// Unknown inherited values may contain credentials under arbitrary names.+// Explicitly configured values remain private, including overrides of these keys.+const PUBLIC_ENV_KEYS = new Set([+ "PATH", "PATHEXT", "SYSTEMROOT", "WINDIR", "COMSPEC", "HOME", "USERPROFILE",+ "HOMEDRIVE", "HOMEPATH", "USER", "USERNAME", "LOGNAME", "SHELL", "LANG",+ "LANGUAGE", "LC_ALL", "LC_CTYPE", "TZ", "TMPDIR", "TEMP", "TMP", "NODE_ENV",+ "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME",+ "PWD", "OLDPWD", "INIT_CWD", "SHLVL", "_", "TERM", "TERM_PROGRAM", "TERM_PROGRAM_VERSION", "COLORTERM",+ "NPM_LIFECYCLE_EVENT", "NPM_COMMAND", "NPM_EXEC_PATH", "NPM_EXECPATH", "NPM_NODE_EXECPATH",+ "NPM_PACKAGE_NAME", "NPM_PACKAGE_VERSION", "NPM_PACKAGE_JSON", "NPM_CONFIG_USER_AGENT",+ "XPC_SERVICE_NAME", "VITEST_POOL_ID", "VITEST_WORKER_ID",+ "PAPERCLIP_AGENT_ID", "PAPERCLIP_COMPANY_ID", "PAPERCLIP_RUN_ID", "PAPERCLIP_TASK_ID",+]);+const PUBLIC_BOOLEAN_ENV_KEYS = new Set([+ "OPENCODE_ALLOW_ALL_MODELS", "CLAUDE_CODE_USE_BEDROCK", "GOOGLE_GENAI_USE_GCA",+ "CI", "NO_COLOR", "FORCE_COLOR",+ "VITEST", "MALLOCNANOZONE", "CODEX_CI", "CODEX_SHELL", "CODEX_SAGE_BACKFILL_TRACKER_TAB_REUSE",+ "DEV", "PROD", "SSR", "PAPERCLIP_REQUIRE_SENTRY_TEST_SDK",+ "PAPERCLIP_DB_BACKUP_ENABLED", "PAPERCLIP_AUTH_DISABLE_SIGN_UP", "PAPERCLIP_ENABLE_COMPANY_DELETION",+ "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "PAPERCLIP_SECRETS_STRICT_MODE",+ "SERVE_UI", "PAPERCLIP_UI_DEV_MIDDLEWARE", "HEARTBEAT_SCHEDULER_ENABLED", "PAPERCLIP_ANNOUNCEMENTS_ENABLED",+ "GITHUB_ACTIONS", "GITHUB_REF_PROTECTED", "RUNNER_DEBUG", "ACTIONS_RUNNER_DEBUG", "ACTIONS_STEP_DEBUG",+]);+const PUBLIC_NUMBER_ENV_KEYS = new Set([+ "PORT", "PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES", "PAPERCLIP_DB_BACKUP_RETENTION_DAYS",+ "PAPERCLIP_WORKSPACE_REAPER_COOLDOWN_DAYS", "HEARTBEAT_SCHEDULER_INTERVAL_MS",+ "GITHUB_RUN_ATTEMPT", "GITHUB_RUN_NUMBER", "GITHUB_RUN_ID", "GITHUB_RETENTION_DAYS",+]);++/** Include declared bindings and unknown values, not just credential-like keys. */+export function collectRunFailureSecretValues(+ env: unknown, secretKeys: Iterable<string> = [], inherited = false,+): string[] {+ if (!env || typeof env !== "object") return [];+ const strings = Object.fromEntries(Object.entries(env).filter(+ (entry): entry is [string, string] => typeof entry[1] === "string",+ ));+ const declared = new Set(secretKeys);+ const values: string[] = [];+ for (const [key, value] of Object.entries(strings)) {+ if (!value) continue;+ const publicSetting = (inherited && (+ PUBLIC_ENV_KEYS.has(key.toUpperCase()) ||+ (PUBLIC_NUMBER_ENV_KEYS.has(key.toUpperCase()) && /^\d+$/.test(value)) ||+ (key === "BASE_URL" && value === "/") ||+ (key === "MODE" && /^(?:test|development|production)$/.test(value))+ )) ||+ (PUBLIC_BOOLEAN_ENV_KEYS.has(key.toUpperCase()) && /^(?:0|1|true|false)$/i.test(value));+ if (declared.has(key) || !publicSetting) values.push(value);+ try {+ const password = new URL(value).password;+ if (password) values.push(value, password, decodeURIComponent(password));+ } catch { /* ordinary environment values are not URLs */ }+ }+ return [...new Set(values)].sort((a, b) => b.length - a.length);+}++/** Redact literal, JSON-escaped and URI-encoded forms without rewriting markers. */+export function redactRunFailureSecretValues<T>(input: T, values: readonly string[]): T {+ const forms = [...new Set(values.filter(Boolean).flatMap((value) => {+ const forms = [value, JSON.stringify(value).slice(1, -1)];+ try { forms.push(encodeURIComponent(value)); } catch { /* malformed Unicode */ }+ return forms;+ }))].sort((a, b) => b.length - a.length);+ if (forms.length === 0) return input;+ const pattern = new RegExp(forms.map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"), "gu");+ const redact = (value: unknown): unknown => {+ if (typeof value === "string") return value.replace(pattern, () => REDACTED_EVENT_VALUE);+ if (Array.isArray(value)) return value.map(redact);+ if (value && typeof value === "object") {+ return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, redact(entry)]));+ }+ return value;+ };+ return redact(input) as T;+}++/** Snapshot only diagnostics. Never walk a request, response, config or prompt. */+export function collectRunFailureDiagnostics(run: Run, options: RunFailureReportOptions): RunFailureDiagnostics {+ const execution = scalars(run, [+ "runtimeMode", "executionStage", "nativePhase", "driverKind", "driverVersion",+ ]);+ if (options.phase) execution.failurePhase = options.phase;+ if (run.startedAt && run.finishedAt) {+ const durationMs = run.finishedAt.getTime() - run.startedAt.getTime();+ if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;+ }+ const result = run.resultJson;+ Object.assign(execution, scalars(result, [+ "mode", "stopReason", "timeoutFired", "timeoutSource", "timeoutConfigured",+ "effectiveTimeoutSec", "errorFamily",+ ]));+ const adapter = scalars(options.adapterErrorMeta, [+ "category", "phase", "errorName", "acpCode", "causeMessage", "retryable",+ "stackPreview", "status", "statusCode", "requestId",+ ]);+ const provider = scalars(read(result, "terminalSessionFailure"), ["category", "title", "details"]);+ const truncatedFields: string[] = [];+ const providerTruncation = read(read(result, "terminalSessionFailure"), "truncatedFields");+ if (Array.isArray(providerTruncation)) {+ for (const field of ["title", "details"]) {+ if (providerTruncation.includes(field)) truncatedFields.push(`provider.${field}`);+ }+ }+ const exceptions: RunFailureException[] = [];+ const seen = new Set<unknown>();+ let error = options.error;+ while (error !== undefined && error !== null && exceptions.length < 4) {+ if (seen.has(error)) {+ truncatedFields.push("exceptions.cycle");+ break;+ }+ seen.add(error);+ if (typeof error === "string") {+ exceptions.push({ message: error });+ error = undefined;+ break;+ }+ if (typeof error !== "object") break;+ const entry: RunFailureException = {};+ for (const field of ["name", "message", "stack", "code"] as const) {+ const value = read(error, field);+ if (typeof value === "string") entry[field] = value;+ else if (field === "code" && typeof value === "number" && Number.isFinite(value)) entry.code = String(value);+ }+ const status = read(error, "status") ?? read(error, "statusCode");+ if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) entry.status = status;+ const requestId = read(error, "requestId") ?? read(error, "request_id");+ if (typeof requestId === "string") entry.requestId = requestId;+ if (Object.keys(entry).length > 0) exceptions.push(entry);+ else break;+ error = read(error, "cause");+ }+ if (error !== undefined && error !== null && exceptions.length === 4) truncatedFields.push("exceptions.depth");+ return { execution, adapter, provider, exceptions, truncatedFields };+}++/** Redact complete values before cutting, including credentials across a cut. */+export function sanitizeRunFailureText(input: string, limit: number): string {+ const normalized = input+ .replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")+ .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "")+ .replace(/[\ud800-\udfff]/gu, "\ufffd");+ const clean = redactSensitiveText(redactDiagnosticText(redactCurrentUserText(normalized)));+ if (clean.length <= limit) return clean;+ const suffix = "\n[truncated]";+ let end = limit - suffix.length;+ if ((clean.codePointAt(end - 1) ?? 0) > 0xffff) end--;+ return clean.slice(0, end) + suffix;+}++/** Called after the run's registered secret values have also been removed. */+export function sanitizeRunFailureDiagnostics(raw: RunFailureDiagnostics): RunFailureDiagnostics {+ const truncatedFields = [...raw.truncatedFields];+ const text = (value: string, path: string, limit: number) => {+ const sanitized = sanitizeRunFailureText(value, limit);+ if (sanitized.endsWith("\n[truncated]")) truncatedFields.push(path);+ return sanitized;+ };+ const context = (value: Context, prefix: string, limits: Record<string, number> = {}): Context =>+ Object.fromEntries(Object.entries(value).map(([key, value]) => [key,+ typeof value === "string" ? text(value, `${prefix}.${key}`, limits[key] ?? 200) : value,+ ]));+ return {+ execution: context(raw.execution, "execution"),+ adapter: context(raw.adapter, "adapter", { causeMessage: 2048, stackPreview: 8192 }),+ provider: context(raw.provider, "provider", { title: 4096, details: 12288 }),+ exceptions: raw.exceptions.map((entry, i) => context(entry as Context, `exceptions.${i}`, { message: 2048, stack: 8192 })),+ truncatedFields: [...new Set(truncatedFields)],+ };+}diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.tsindex 7e40345513..0c044b6615 100644--- a/server/src/services/run-failure-report.ts+++ b/server/src/services/run-failure-report.ts@@ -1,8 +1,14 @@-import { eq } from "drizzle-orm";+import { and, eq } from "drizzle-orm";import { agents, heartbeatRuns, type Db } from "@paperclipai/db";import { captureRunFailure, type RunFailureStatus } from "../sentry.js";-import { redactCurrentUserText } from "../log-redaction.js";-import { redactSensitiveText } from "../redaction.js";+import {+ collectRunFailureDiagnostics,+ collectRunFailureSecretValues,+ redactRunFailureSecretValues,+ sanitizeRunFailureDiagnostics,+ sanitizeRunFailureText,+ type RunFailureReportOptions,+} from "./run-failure-diagnostics.js";import { logger } from "../middleware/logger.js";type HeartbeatRun = typeof heartbeatRuns.$inferSelect;@@ -23,15 +29,6 @@ const pendingRunFailureReports = new Set<Promise<void>>();/** Bounds the shutdown wait, so one stuck report cannot hang the process exit. */const PENDING_REPORT_DRAIN_TIMEOUT_MS = 5_000;-/**- * Remove a credential and the current user's home path from adapter-supplied- * text, then cut the result to `maxLength`. Run the length cut after the- * redaction, so a credential cannot survive at a cut boundary.- */-function sanitizeAdapterText(input: string, maxLength: number): string {- return redactSensitiveText(redactCurrentUserText(input)).slice(0, maxLength);-}-function isRunFailureStatus(status: string): status is RunFailureStatus {return status === "failed" || status === "timed_out";}@@ -53,10 +50,10 @@ function readTaskId(run: HeartbeatRun): string | null {* its own in-flight promise, so a caller that does not await it still lets* shutdown find and wait for the report β see `waitForPendingRunFailureReports`.*/-export function reportRunFailure(db: Db, run: HeartbeatRun): Promise<void> {+export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureReportOptions = {}): Promise<void> {if (!isRunFailureStatus(run.status)) return Promise.resolve();const runStatus = run.status;- const report = captureTerminalRunFailure(db, run, runStatus);+ const report = captureTerminalRunFailure(db, run, runStatus, options);pendingRunFailureReports.add(report);void report.finally(() => pendingRunFailureReports.delete(report));return report;@@ -66,12 +63,21 @@ async function captureTerminalRunFailure(db: Db,run: HeartbeatRun,runStatus: RunFailureStatus,+ options: RunFailureReportOptions,): Promise<void> {try {+ const snapshot = redactRunFailureSecretValues({+ errorMessage: run.error ?? "",+ errorCode: run.errorCode ?? null,+ diagnostics: collectRunFailureDiagnostics(run, options),+ }, [...new Set([+ ...collectRunFailureSecretValues(process.env, [], true),+ ...(options.secretValues ?? []),+ ])].sort((a, b) => b.length - a.length));const agent = await db.select({ adapterType: agents.adapterType }).from(agents)- .where(eq(agents.id, run.agentId))+ .where(and(eq(agents.id, run.agentId), eq(agents.companyId, run.companyId))).then((rows) => rows[0] ?? null);const taskId = readTaskId(run);@@ -80,18 +86,26 @@ async function captureTerminalRunFailure(return;}+ // Resolve registered values before truncation. A failed resolution must+ // not send an incompletely redacted report.+ let redacted = snapshot;+ if (Array.isArray(run.contextSnapshot?.paperclipSecretRedactions)) {+ const { createRunSecretRedactionRegistry } = await import("./run-secret-redaction.js");+ redacted = await createRunSecretRedactionRegistry(db).redactForRun(run.companyId, run.id, snapshot);+ }captureRunFailure({taskId,runId: run.id,- errorMessage: sanitizeAdapterText(run.error ?? "", MAX_ERROR_MESSAGE_LENGTH),+ errorMessage: sanitizeRunFailureText(redacted.errorMessage, MAX_ERROR_MESSAGE_LENGTH),errorCode:- run.errorCode === null+ redacted.errorCode === null? null- : sanitizeAdapterText(run.errorCode, MAX_ERROR_CODE_LENGTH),+ : sanitizeRunFailureText(redacted.errorCode, MAX_ERROR_CODE_LENGTH),agentAdapter: agent?.adapterType ?? UNKNOWN_ADAPTER,runStatus,exitCode: run.exitCode,signal: run.signal,+ diagnostics: sanitizeRunFailureDiagnostics(redacted.diagnostics),});} catch (err) {logger.warn({ err, runId: run.id }, "failed to report run failure to Sentry");