Skip to content

The open-source app everyone uses to manage agents at work

66.3 KB
Raw Blame
import { randomUUID } from "node:crypto";
import os from "node:os";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, applyPendingMigrations, authUsers, closeRegisteredClients, companies, companySecrets, createDb, dotAgentBindings,
  dotRunnerAssignments, grokBotDispatches, grokBotRoutines, heartbeatRuns, issues, mcpOauthClients, mcpOauthGrants, nativeRunFinalizations,
  type Db } from "@paperclipai/db";
import {
  getEmbeddedPostgresTestSupport,
  startEmbeddedPostgresTestDatabase,
} from "../../__tests__/helpers/embedded-postgres.js";

const mockCaptureRunFailure = vi.hoisted(() => vi.fn());
const mockRedactCurrentUserText = vi.hoisted(() => vi.fn());
const mockResolveSecret = vi.hoisted(() => vi.fn());

vi.mock("../../secrets/provider-registry.js", () => ({
  getSecretProvider: () => ({ resolveVersion: mockResolveSecret }),
}));

vi.mock("../../sentry.js", () => ({
  captureRunFailure: mockCaptureRunFailure,
}));
// Wrap the real function instead of a fake, so tests can assert the actual
// redacted output while still spying on the call. A fake output would hide
// whether the composed redaction in run-failure-report.ts is correct.
vi.mock("../../log-redaction.js", async (importOriginal) => {
  const actual = await importOriginal<typeof import("../../log-redaction.js")>();
  mockRedactCurrentUserText.mockImplementation(actual.redactCurrentUserText);
  return { ...actual, redactCurrentUserText: mockRedactCurrentUserText };
});

import { isNewTerminalRunFailure, reportRunFailure, waitForPendingRunFailureReports } from "../run-failure-report.js";
import { controllerLeaseLossCancellation } from "../run-cancellation.js";
import { REDACTED_EVENT_VALUE } from "../../redaction.js";

const testDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL;
const embeddedPostgresSupport = testDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;

describeEmbeddedPostgres("reportRunFailure", () => {
  let db!: ReturnType<typeof createDb>;
  let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
  let companyId!: string;
  let agentId!: string;
  let inheritedEnv: NodeJS.ProcessEnv;

  beforeAll(async () => {
    if (testDatabaseUrl) {
      await applyPendingMigrations(testDatabaseUrl);
      db = createDb(testDatabaseUrl);
    } else {
      tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-failure-report-");
      db = createDb(tempDb.connectionString);
    }
  }, 20_000);

  beforeEach(() => {
    // Unknown host values intentionally count as secrets, including short values
    // such as "1". Keep these fixtures independent of the developer/CI environment
    // and add secret values explicitly in the tests that exercise redaction.
    inheritedEnv = process.env;
    process.env = Object.fromEntries(
      ["PATH", "HOME", "USER", "USERNAME", "LOGNAME", "USERPROFILE", "TMPDIR", "TEMP", "TMP"]
        .flatMap((key) => inheritedEnv[key] === undefined ? [] : [[key, inheritedEnv[key]]]),
    );
    process.env.PAPERCLIP_SENTRY_RUN_FAILURES = "unexpected";
  });

  afterEach(async () => {
    vi.clearAllMocks();
    vi.unstubAllEnvs();
    process.env = inheritedEnv;
    await db.delete(grokBotRoutines);
    await db.delete(companySecrets);
    await db.delete(nativeRunFinalizations);
    await db.delete(heartbeatRuns);
    await db.delete(issues);
    await db.delete(agents);
    await db.delete(companies);
    await db.delete(mcpOauthClients);
    await db.delete(authUsers);
  });

  afterAll(async () => {
    if (testDatabaseUrl) await closeRegisteredClients(testDatabaseUrl);
    await tempDb?.cleanup();
  });

  async function seedCompanyAndAgent(agentOverrides: Partial<typeof agents.$inferInsert> = {}) {
    companyId = randomUUID();
    agentId = randomUUID();
    await db.insert(companies).values({
      id: companyId,
      name: "Paperclip",
      issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
      requireBoardApprovalForNewAgents: false,
      defaultResponsibleUserId: "responsible-user",
    });
    await db.insert(agents).values({
      id: agentId,
      companyId,
      name: "CodexCoder",
      role: "engineer",
      status: "active",
      adapterType: "codex_local",
      adapterConfig: {},
      runtimeConfig: {},
      permissions: {},
      ...agentOverrides,
    });
    return { companyId, agentId };
  }

  function buildRun(
    overrides: Partial<typeof heartbeatRuns.$inferSelect> = {},
  ): typeof heartbeatRuns.$inferSelect {
    return {
      id: randomUUID(),
      companyId,
      agentId,
      status: "failed",
      error: "the provider process exited with code 1",
      errorCode: "process_lost",
      nativeIssueId: randomUUID(),
      contextSnapshot: null,
      ...overrides,
    } as unknown as typeof heartbeatRuns.$inferSelect;
  }

  it.each([undefined, "", "off", "true", "invalid"])("does no reporting work without an explicit valid mode: %s", async mode => {
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", mode);
    if (mode === undefined) delete process.env.PAPERCLIP_SENTRY_RUN_FAILURES;
    const select = vi.fn();
    await reportRunFailure({ select } as unknown as Db, buildRun());
    expect(select).not.toHaveBeenCalled();
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it.each(["quota", "git", "secret", "selection", "workspace", "hermes"])("reports known %s failures in all mode", async kind => {
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
    await seedCompanyAndAgent(kind === "hermes" ? { adapterType: "hermes_gateway" } : {});
    const fixtures = {
      quota: providerQuotaRun,
      git: gitConnectionRun,
      secret: missingSecretRun,
      selection: aiSelectionRun,
      workspace: localPathConfigurationRun,
      hermes: hermesConnectionRun,
    };
    const fixture = fixtures[kind as keyof typeof fixtures]();
    const phase = ["git", "secret", "selection", "workspace"].includes(kind) ? "setup" : "execute";
    const before = structuredClone(fixture);
    await reportRunFailure(db, fixture, { phase });
    expect(mockCaptureRunFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
      companyId, agentId, runId: fixture.id, runStatus: "failed", errorCode: fixture.errorCode,
    }));
    expect(fixture).toEqual(before);
  });

  it("reports company-scoped runs that have no task", async () => {
    await seedCompanyAndAgent();
    const fixture = buildRun({ nativeIssueId: null, contextSnapshot: null });
    await reportRunFailure(db, fixture);
    expect(mockCaptureRunFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
      companyId, agentId, taskId: null, runId: fixture.id,
    }));
  });

  it.each(["succeeded", "queued", "running", "interrupted", "cancelled"])("all mode excludes %s runs and expected operator stops", async status => {
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
    const select = vi.fn();
    await reportRunFailure({ select } as unknown as Db, buildRun({ status, startedAt: new Date(), resultJson: {
      cancellation: { source: "operator", expected: true, initiator: { type: "user" }, reason: "Stopped", recordedAt: new Date().toISOString() },
    } }));
    expect(select).not.toHaveBeenCalled();
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it.each(["fresh", "resume", "provider_started"] as const)("keeps workspace base-ref observations reportable: %s", async mode => {
    await seedCompanyAndAgent();
    const run = buildRun({ errorCode: "configuration_incomplete", executionStage: "preparing", exitCode: null, signal: null,
      runtimeMode: mode === "resume" ? "native" : "legacy", runnerProfileJson: mode === "resume" ? { nativeExecutionInput: {} } : null,
      resultJson: { configurationIncomplete: { reason: "workspace_base_ref_unresolved", baseRefDiagnostic: {
        schemaVersion: 1, remoteLookup: "resolved", authLookup: "resolved", fetch: "failed", fetchExitCode: 128,
        fetchFailureKind: "remote_ref_not_found", refResolution: "failed", refExitCode: 128,
      } }, executionRecovery: { kind: "bootstrap", providerWorkStarted: mode === "provider_started" } },
    });
    const before = structuredClone(run);
    await reportRunFailure(db, run, { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ diagnostics: expect.objectContaining({
      execution: expect.objectContaining({ workspaceBaseRefFetch: "failed", workspaceBaseRefFetchFailureKind: "remote_ref_not_found" }),
    }) }));
    expect(run).toEqual(before);
  });

  const gitConnection = { schemaVersion: 1, provider: "git", operation: "clone", reason: "authentication_failed" };
  const hermesConnection = { schemaVersion: 1, provider: "hermes_gateway", operation: "create_run", reason: "connection_refused" };

  function gitConnectionRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      errorCode: "setup_failed", executionStage: "preparing", exitCode: null, signal: null,
      resultJson: { connectionFailure: gitConnection,
        executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
      ...overrides,
    });
  }

  function hermesConnectionRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      errorCode: "hermes_gateway_connect_failed", executionStage: "executing", exitCode: 1, signal: null,
      resultJson: { connectionFailure: hermesConnection }, ...overrides,
    });
  }

  it("keeps a proven Git clone failure and its persisted task error unchanged", async () => {
    await seedCompanyAndAgent();
    const input = gitConnectionRun();
    const [run] = await db.insert(heartbeatRuns).values(input).returning();
    const before = structuredClone(run);
    await reportRunFailure(db, run!, { phase: "setup" });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    expect(run).toEqual(before);
    expect(await db.select().from(heartbeatRuns)).toEqual([before]);
  });

  it("keeps an all-external Git materialization aggregate local even across different external causes", async () => {
    await seedCompanyAndAgent();
    const run = gitConnectionRun({ errorCode: "workspace_validation_failed" });
    run.resultJson!.workspaceValidation = { reason: "git_worktree_base_materialization_failed", materializationFailures: [
      { connectionFailure: gitConnection },
      { connectionFailure: { ...gitConnection, reason: "repository_unavailable" } },
    ] };
    const before = structuredClone(run);
    await reportRunFailure(db, run, { phase: "setup" });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    expect(run).toEqual(before);
  });

  it.each([
    { status: "timed_out" }, { executionStage: "executing" }, { executionStage: null },
    { errorCode: "adapter_failed" }, { exitCode: 1 }, { exitCode: 0 }, { signal: "SIGTERM" },
    { resultJson: {} },
    { resultJson: { connectionFailure: { ...gitConnection, schemaVersion: 2 }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } },
    { resultJson: { connectionFailure: hermesConnection, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } },
  ])("reports a Git marker without its matching setup outcome: %j", async (overrides) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, gitConnectionRun(overrides), { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it.each([undefined, "execute"] as const)("reports Git failures outside setup: %s", async (phase) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, gitConnectionRun(), { phase });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it.each([undefined, { kind: "bootstrap" }, { kind: "bootstrap", providerWorkStarted: true },
    { kind: "runtime", providerWorkStarted: false }])("reports Git failures without unstarted bootstrap proof: %j", async (recovery) => {
    await seedCompanyAndAgent();
    const run = gitConnectionRun();
    run.resultJson!.executionRecovery = recovery;
    await reportRunFailure(db, run, { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it.each([
    [], [null], [{ connectionFailure: gitConnection }, {}],
    [{ connectionFailure: gitConnection }, { connectionFailure: hermesConnection }],
    [{ connectionFailure: gitConnection }, { connectionFailure: { ...gitConnection, reason: "unknown" } }],
    [{ connectionFailure: { ...gitConnection, reason: "repository_unavailable" } }],
  ].map((failures) => ({ failures })))("reports mixed, malformed, or inconsistent Git materialization evidence: %j", async ({ failures }) => {
    await seedCompanyAndAgent();
    const run = gitConnectionRun({ errorCode: "workspace_validation_failed" });
    run.resultJson!.workspaceValidation = { reason: "git_worktree_base_materialization_failed", materializationFailures: failures };
    await reportRunFailure(db, run, { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it.each([
    { reason: "git_worktree_base_materialization_failed", materializationFailures: [{ connectionFailure: gitConnection }] },
    { reason: "git_worktree_not_reusable" },
  ])("does not use setup_failed to bypass separate workspace validation evidence: %j", async (validation) => {
    await seedCompanyAndAgent();
    const run = gitConnectionRun();
    run.resultJson!.workspaceValidation = validation;
    await reportRunFailure(db, run, { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it.each([
    ["configuration", "endpoint_missing", "hermes_gateway_api_base_url_missing"],
    ["configuration", "endpoint_invalid", "hermes_gateway_api_base_url_invalid"],
    ["configuration", "insecure_transport", "hermes_gateway_plain_http_remote_denied"],
    ["configuration", "credentials_missing", "hermes_gateway_api_key_missing"],
    ["create_run", "authentication_failed", "hermes_gateway_auth_failed"],
    ["create_run", "endpoint_not_found", "hermes_gateway_runs_unsupported"],
    ["create_run", "rate_limited", "hermes_gateway_rate_limited"],
    ["create_run", "remote_unavailable", "hermes_gateway_upstream_error"],
    ...["connection_refused", "dns_failure", "network_unreachable", "connection_reset", "connection_timeout", "tls_failure"]
      .flatMap((reason) => ["hermes_gateway_connect_failed", "hermes_gateway_protocol_error"].map((code) => ["create_run", reason, code])),
  ])("keeps the matching Hermes %s/%s task error local", async (operation, reason, errorCode) => {
    await seedCompanyAndAgent({ adapterType: "hermes_gateway" });
    const run = hermesConnectionRun({ errorCode, resultJson: { connectionFailure: { ...hermesConnection, operation, reason } } });
    const before = structuredClone(run);
    await reportRunFailure(db, run, { phase: "execute" });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    expect(run).toEqual(before);
  });

  it.each([undefined, { kind: "bootstrap", providerWorkStarted: true }, { kind: "runtime", providerWorkStarted: true }])(
    "does not infer remote nonacceptance from a classified Hermes connection failure: %j", async (recovery) => {
      await seedCompanyAndAgent({ adapterType: "hermes_gateway" });
      const input = hermesConnectionRun();
      input.resultJson!.executionRecovery = recovery;
      const [run] = await db.insert(heartbeatRuns).values(input).returning();
      const before = structuredClone(run);
      await reportRunFailure(db, run!);
      expect(mockCaptureRunFailure).not.toHaveBeenCalled();
      expect(await db.select().from(heartbeatRuns)).toEqual([before]);
    },
  );

  it.each([
    { status: "timed_out" }, { status: "cancelled", startedAt: new Date(0) },
    { exitCode: 0 }, { exitCode: null }, { signal: "SIGTERM" },
    { errorCode: "hermes_gateway_timeout" }, { errorCode: "hermes_gateway_run_failed" },
    { errorCode: "hermes_gateway_auth_failed" },
    { resultJson: {} },
    { resultJson: { connectionFailure: { ...hermesConnection, reason: "unknown" } } },
    { resultJson: { connectionFailure: { ...hermesConnection, privateUrl: "https://private.example.test" } } },
    { resultJson: { connectionFailure: gitConnection } },
    { resultJson: { connectionFailure: { ...hermesConnection, operation: "configuration", reason: "credentials_missing" } } },
  ])("reports an unproven or mismatched Hermes outcome: %j", async (overrides) => {
    await seedCompanyAndAgent({ adapterType: "hermes_gateway" });
    const run = hermesConnectionRun(overrides);
    const before = structuredClone(run);
    await reportRunFailure(db, run, { phase: "execute" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
    expect(run).toEqual(before);
  });

  it("keeps setup errors reportable even when they carry a matching Hermes marker", async () => {
    await seedCompanyAndAgent({ adapterType: "hermes_gateway" });
    await reportRunFailure(db, hermesConnectionRun(), { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it.each(["codex_local", "hermes_local"])("requires the actual database adapter, not a Hermes marker on %s", async (adapterType) => {
    await seedCompanyAndAgent({ adapterType });
    const run = hermesConnectionRun({ contextSnapshot: { adapterType: "hermes_gateway" } });
    await reportRunFailure(db, run);
    expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ agentAdapter: adapterType }));
  });

  it.each(["missing", "other_company"])("does not suppress Hermes when its database agent is %s", async (kind) => {
    await seedCompanyAndAgent({ adapterType: "hermes_gateway" });
    const run = hermesConnectionRun(kind === "missing" ? { agentId: randomUUID() } : { companyId: randomUUID() });
    await reportRunFailure(db, run);
    expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({ agentAdapter: "unknown" }));
  });

  function providerQuotaRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      error: "ACP agent reported a terminal limit failure.\nYou've hit your session limit · resets 2am (UTC)",
      errorCode: "provider_quota", executionStage: "dispatching", exitCode: 1, signal: null,
      resultJson: { errorFamily: "provider_quota", stopReason: "adapter_failed" }, ...overrides,
    });
  }

  it.each(["claude_local", "codex_local"])("keeps a classified %s provider quota wait local", async (adapterType) => {
    await seedCompanyAndAgent({ adapterType });
    const [run] = await db.insert(heartbeatRuns).values(providerQuotaRun()).returning();
    const before = structuredClone(run);
    for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, run!, { phase });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    expect(await db.select().from(heartbeatRuns)).toEqual([before]);
  });

  it.each([
    { status: "timed_out" }, { status: "cancelled", startedAt: new Date(0) }, { signal: "SIGTERM" },
    { errorCode: "adapter_failed" }, { errorCode: "claude_transient_upstream" },
    { resultJson: {} }, { resultJson: null }, { resultJson: { errorFamily: "transient_upstream" } },
  ])("reports a provider quota outcome without matching adapter evidence: %j", async (overrides) => {
    await seedCompanyAndAgent({ adapterType: "claude_local" });
    await reportRunFailure(db, providerQuotaRun(overrides), { phase: "execute" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it("keeps a provider quota code reportable during setup", async () => {
    await seedCompanyAndAgent({ adapterType: "claude_local" });
    await reportRunFailure(db, providerQuotaRun(), { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  function codexModelCapacityRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      error: "ACP agent reported a terminal service failure.\nSelected model is at capacity. Please try a different model.",
      errorCode: "codex_transient_upstream", executionStage: "dispatching", exitCode: 1, signal: null,
      resultJson: {
        errorFamily: "transient_upstream", stopReason: "adapter_failed",
        terminalSessionFailure: { category: "service", title: "Selected model is at capacity. Please try a different model." },
      },
      ...overrides,
    });
  }

  it("keeps a classified Codex ACP model capacity retry local", async () => {
    await seedCompanyAndAgent({ adapterType: "codex_local" });
    for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, codexModelCapacityRun(), { phase });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it.each([
    { status: "timed_out" }, { signal: "SIGTERM" }, { errorCode: "acpx_turn_failed" },
    { resultJson: { errorFamily: "transient_upstream" } },
    { resultJson: { errorFamily: "provider_quota", terminalSessionFailure: { category: "service" } } },
    { resultJson: { errorFamily: "transient_upstream", terminalSessionFailure: { category: "connection" } } },
  ])("reports a Codex transient outcome without typed capacity evidence: %j", async (overrides) => {
    await seedCompanyAndAgent({ adapterType: "codex_local" });
    await reportRunFailure(db, codexModelCapacityRun(overrides), { phase: "execute" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it("reports a Codex ACP model capacity failure during setup or in all mode", async () => {
    await seedCompanyAndAgent({ adapterType: "codex_local" });
    await reportRunFailure(db, codexModelCapacityRun(), { phase: "setup" });
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
    await reportRunFailure(db, codexModelCapacityRun(), { phase: "execute" });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(2);
  });

  function providerAccessRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      error: "ACP agent reported a terminal access failure.\nInvalid API key · Fix external API key",
      errorCode: "claude_auth_required", executionStage: "dispatching", exitCode: 1, signal: null,
      resultJson: {
        stopReason: "adapter_failed",
        terminalSessionFailure: { category: "access", title: "Invalid API key · Fix external API key" },
      },
      ...overrides,
    });
  }

  it.each(["claude_auth_required", "acpx_auth_required"])("keeps a typed ACP provider access failure (%s) local", async (errorCode) => {
    await seedCompanyAndAgent({ adapterType: "claude_local" });
    for (const phase of [undefined, "execute"] as const) await reportRunFailure(db, providerAccessRun({ errorCode }), { phase });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it.each([
    { status: "timed_out" }, { signal: "SIGTERM" }, { errorCode: "acpx_turn_failed" }, { errorCode: "adapter_failed" },
    { resultJson: {} }, { resultJson: null }, { resultJson: { terminalSessionFailure: { category: "service" } } },
    { resultJson: { terminalSessionFailure: "access" } },
  ])("reports a provider auth outcome without typed ACP access evidence: %j", async (overrides) => {
    await seedCompanyAndAgent({ adapterType: "claude_local" });
    await reportRunFailure(db, providerAccessRun(overrides), { phase: "execute" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it("reports a typed ACP provider access failure during setup or in all mode", async () => {
    await seedCompanyAndAgent({ adapterType: "claude_local" });
    await reportRunFailure(db, providerAccessRun(), { phase: "setup" });
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
    await reportRunFailure(db, providerAccessRun(), { phase: "execute" });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(2);
  });

  function restartInterruptedSetupRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      error: "Process lost -- server may have restarted; retrying once",
      runtimeMode: "legacy", executionStage: "preparing", processPid: null, processGroupId: null,
      resultJson: {
        executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
        processLossDiagnostic: {
          pidRecorded: false, groupRecorded: false, localCheck: "no_identifiers",
          retryEligible: true, runPredatesObserver: true,
        },
      },
      ...overrides,
    });
  }

  it("keeps a retried restart interruption of legacy setup local", async () => {
    await seedCompanyAndAgent({ adapterType: "paperclip_runner" });
    await reportRunFailure(db, restartInterruptedSetupRun());
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it("reports a retried restart interruption of legacy setup in all mode", async () => {
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
    await seedCompanyAndAgent({ adapterType: "paperclip_runner" });
    await reportRunFailure(db, restartInterruptedSetupRun());
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it.each([
    { executionStage: "dispatching" }, { runtimeMode: "native" }, { processPid: 4242 },
    { errorCode: "adapter_failed" },
    { resultJson: { processLossDiagnostic: { localCheck: "no_identifiers", retryEligible: true, runPredatesObserver: true } } },
    { resultJson: {
      executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
      processLossDiagnostic: { localCheck: "no_identifiers", retryEligible: false, runPredatesObserver: true },
    } },
    { resultJson: {
      executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
      processLossDiagnostic: { localCheck: "no_identifiers", retryEligible: true, runPredatesObserver: false },
    } },
  ])("reports process loss without restart-before-dispatch evidence: %j", async (overrides) => {
    await seedCompanyAndAgent({ adapterType: "paperclip_runner" });
    await reportRunFailure(db, restartInterruptedSetupRun(overrides));
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  async function expiredGrokOffer(input: {
    wakes: Array<{ status: "rejected" | "accepted" | "acknowledged" | "unknown"; httpStatus: number | null }>;
    acceptedAt?: Date | null;
    originalFailureCode?: string;
    finalization?: boolean;
    offers?: number;
  }) {
    await seedCompanyAndAgent({ adapterType: "paperclip_runner", adapterConfig: { provider: "grok_bot" } });
    const userId = randomUUID();
    await db.insert(authUsers).values({ id: userId, name: "Operator", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
    await db.insert(mcpOauthClients).values({ id: userId, name: "Grok", redirectUris: ["https://client.example/callback"] });
    const [grant] = await db.insert(mcpOauthGrants).values({ companyId, userId, clientId: userId, resource: "https://paperclip.example/mcp/runner",
      purpose: "agent", agentId, scopes: ["paperclip:agent"] }).returning();
    const [binding] = await db.insert(dotAgentBindings).values({ companyId, agentId, operatorId: userId, grantId: grant!.id,
      providerKind: "grok_bot", status: "ready" }).returning();
    const [secret] = await db.insert(companySecrets).values({ companyId, key: `grok-${binding!.id}`, name: "Grok routine" }).returning();
    const [routine] = await db.insert(grokBotRoutines).values({ companyId, agentId, bindingId: binding!.id, bindingGeneration: 1,
      botName: "Grok", secretId: secret!.id, secretVersion: 1 }).returning();
    const [issue] = await db.insert(issues).values({ companyId, title: "Grok task", assigneeAgentId: agentId, status: "in_progress" }).returning();
    const run = buildRun({ errorCode: "native_provider_terminal_failed", error: "Dot assignment acceptance or execution authority expired.",
      nativeIssueId: issue!.id, runtimeMode: "native", executionStage: "preparing", nativePhase: "terminal_failure", exitCode: null, signal: null });
    await db.insert(heartbeatRuns).values(run);
    if (input.finalization !== false) {
      await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId, issueId: run.nativeIssueId!, phase: "terminal_failure",
        failureCode: "native_provider_terminal_failed", failureDetail: { originalFailureCode: input.originalFailureCode ?? "dot_assignment_expired" } });
    }
    const offers = await db.insert(dotRunnerAssignments).values(Array.from({ length: input.offers ?? 1 }, () => ({ companyId, agentId,
      bindingId: binding!.id, bindingGeneration: 1, runId: run.id, normalizedSessionId: randomUUID(), turnId: randomUUID(), controllerGeneration: 1,
      catalogDigest: "fixture", projection: {}, status: "fenced" as const, acceptBy: new Date(Date.now() - 1000),
      expiresAt: new Date(Date.now() + 3_600_000), acceptedAt: input.acceptedAt ?? null }))).returning();
    const assignment = offers[0];
    for (const wake of input.wakes) {
      await db.insert(grokBotDispatches).values({ companyId, routineId: routine!.id, bindingId: binding!.id, bindingGeneration: 1,
        grantId: grant!.id, routineRevision: 1, assignmentId: assignment!.id, sourceEventId: randomUUID(), kind: "mailbox_updated",
        payload: {}, status: wake.status, httpStatus: wake.httpStatus, expiresAt: new Date(Date.now() + 600_000) });
    }
    return run;
  }

  it("keeps a Grok offer local when the endpoint rejected every work wake with HTTP 4xx", async () => {
    const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: 400 }] });
    await reportRunFailure(db, run, { phase: "execute" });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it("reports a rejected Grok wake in all mode", async () => {
    vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all");
    const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }] });
    await reportRunFailure(db, run);
    expect(mockCaptureRunFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
      companyId, agentId, runId: run.id, errorCode: "native_provider_terminal_failed",
    }));
  });

  it.each([
    ["no wake was sent", { wakes: [] }],
    ["a wake was rejected locally without HTTP evidence", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "rejected", httpStatus: null }] }],
    ["the endpoint accepted a wake", { wakes: [{ status: "rejected", httpStatus: 400 }, { status: "acknowledged", httpStatus: 200 }] }],
    ["delivery is uncertain", { wakes: [{ status: "unknown", httpStatus: 503 }] }],
    ["the bot accepted the work", { wakes: [{ status: "rejected", httpStatus: 400 }], acceptedAt: new Date() }],
    ["the run failed for another provider reason", { wakes: [{ status: "rejected", httpStatus: 400 }], originalFailureCode: "provider_failed" }],
    ["the run has no finalization", { wakes: [{ status: "rejected", httpStatus: 400 }], finalization: false }],
    ["the run has more than one offer", { wakes: [{ status: "rejected", httpStatus: 400 }], offers: 2 }],
  ] as const)("keeps an expired Grok offer reportable when %s", async (_label, input) => {
    const run = await expiredGrokOffer({ ...input, wakes: [...input.wakes] });
    await reportRunFailure(db, run, { phase: "execute" });
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it("keeps an expired Grok offer reportable when its receipts cannot be read", async () => {
    const run = await expiredGrokOffer({ wakes: [{ status: "rejected", httpStatus: 400 }] });
    const select = db.select.bind(db);
    // The agent lookup succeeds; the receipt reads fail.
    const spy = vi.spyOn(db, "select").mockImplementationOnce(select as typeof db.select)
      .mockImplementation(() => { throw new Error("fixture receipt read failed"); });
    try {
      await reportRunFailure(db, run, { phase: "execute" });
    } finally {
      spy.mockRestore();
    }
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  function aiSelectionRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      errorCode: "configuration_incomplete",
      executionStage: "preparing",
      resultJson: {
        configurationIncomplete: {
          reason: "ai_connection_unavailable",
          selectionFailure: "ai_connection_default_missing",
        },
        executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
      },
      ...overrides,
    });
  }

  it.each([
    "ai_connection_responsible_user_missing", "ai_connection_default_missing",
    "ai_connection_missing", "ai_connection_incompatible", "ai_connection_unavailable",
    "ai_connection_credential_not_shared",
  ])("keeps the proven AI selection blocker local: %s", async (selectionFailure) => {
    await seedCompanyAndAgent();
    const run = aiSelectionRun();
    (run.resultJson!.configurationIncomplete as Record<string, unknown>).selectionFailure = selectionFailure;
    const saved = structuredClone(run);
    await reportRunFailure(db, run, { phase: "setup" });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    expect(run).toEqual(saved);
  });

  it.each([undefined, null, {}, [], "", "database_error", "ai_connection_busy"])(
    "reports AI connection wrappers without a known selection cause: %j", async (selectionFailure) => {
      await seedCompanyAndAgent();
      const run = aiSelectionRun();
      (run.resultJson!.configurationIncomplete as Record<string, unknown>).selectionFailure = selectionFailure;
      await reportRunFailure(db, run, { phase: "setup" });
      expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    },
  );

  it.each(["resume", "provider_started", "outside_setup", "unmarked"] as const)(
    "reports credential sharing denials without fresh pre-provider proof: %s", async (cause) => {
      await seedCompanyAndAgent();
      const run = aiSelectionRun();
      (run.resultJson!.configurationIncomplete as Record<string, unknown>).selectionFailure = "ai_connection_credential_not_shared";
      run.error = "This credential is not shared with the responsible user";
      if (cause === "resume") {
        run.runtimeMode = "native";
        run.runnerProfileJson = { nativeExecutionInput: {} };
      }
      if (cause === "provider_started") run.resultJson!.executionRecovery = { kind: "bootstrap", providerWorkStarted: true };
      if (cause === "unmarked") delete (run.resultJson!.configurationIncomplete as Record<string, unknown>).selectionFailure;
      await reportRunFailure(db, run, { phase: cause === "outside_setup" ? "execute" : "setup" });
      expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    },
  );

  it.each([
    { runtimeMode: "native", runnerProfileJson: { nativeExecutionInput: {} } },
    { status: "timed_out" }, { errorCode: "setup_failed" }, { executionStage: "executing" },
    { executionStage: null }, { exitCode: 1 }, { exitCode: 0 }, { signal: "SIGTERM" },
    { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" } } },
    { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } },
    { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "resume", providerWorkStarted: false } } },
    { resultJson: { configurationIncomplete: { reason: "ai_connection_unavailable", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "bootstrap" } } },
    { resultJson: { configurationIncomplete: { reason: "controller_environment_unsupported", selectionFailure: "ai_connection_default_missing" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } },
  ])("reports AI configuration without pre-provider selection proof: %j", async (overrides) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, aiSelectionRun(overrides), { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it.each([undefined, "execute"] as const)("reports AI selection failures outside setup: %s", async (phase) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, aiSelectionRun(), { phase });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  function missingSecretRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      errorCode: "configuration_incomplete",
      executionStage: "preparing",
      resultJson: {
        configurationIncomplete: {
          reason: "secret_binding_missing",
          missingBindings: [{ bindingType: "user_secret_ref", errorCode: "user_secret_missing" }],
        },
        executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
      },
      ...overrides,
    });
  }

  it.each([
    { bindingType: "secret_ref" },
    { bindingType: "secret_ref", errorCode: "binding_missing" },
    ...["binding_missing", "responsible_user_missing", "user_secret_missing", "secret_inactive", "user_secret_definition_inactive"]
      .map((errorCode) => ({ bindingType: "user_secret_ref", errorCode })),
  ])("keeps the known pre-dispatch secret blocker local: %j", async (binding) => {
    await seedCompanyAndAgent();
    const run = missingSecretRun();
    (run.resultJson!.configurationIncomplete as Record<string, unknown>).missingBindings = [binding];
    const saved = structuredClone(run);

    await reportRunFailure(db, run, { phase: "setup" });

    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    expect(run).toEqual(saved);
  });

  it.each([
    {},
    { reason: "secret_binding_missing", missingBindings: [] },
    { reason: "secret_binding_missing", missingBindings: [null] },
    { reason: "secret_binding_missing", missingBindings: [{ errorCode: "user_secret_missing" }] },
    { reason: "secret_binding_missing", missingBindings: [{ bindingType: "user_secret_ref", errorCode: "provider_error" }] },
    { reason: "secret_binding_missing", missingBindings: [{ bindingType: "user_secret_ref", errorCode: "user_secret_definition_missing" }] },
    { reason: "secret_binding_missing", missingBindings: [{ bindingType: "user_secret_ref", errorCode: "new_unknown_reason" }] },
    { reason: "secret_binding_missing", missingBindings: [
      { bindingType: "user_secret_ref", errorCode: "user_secret_missing" },
      { bindingType: "secret_ref", errorCode: "provider_error" },
    ] },
    { reason: "ai_connection_unavailable" },
  ])("reports ambiguous, unknown, and provider failures: %j", async (configurationIncomplete) => {
    await seedCompanyAndAgent();
    const run = missingSecretRun();
    run.resultJson!.configurationIncomplete = configurationIncomplete;

    await reportRunFailure(db, run, { phase: "setup" });

    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it.each([
    { status: "timed_out" },
    { errorCode: "setup_failed" },
    { errorCode: "workspace_validation_failed" },
    { errorCode: "adapter_failed" },
    { executionStage: "executing" },
    { executionStage: null },
    { exitCode: 1 },
    { exitCode: 0 },
    { signal: "SIGTERM" },
  ])("reports missing-secret text without a proven pre-dispatch outcome: %j", async (overrides) => {
    await seedCompanyAndAgent();

    await reportRunFailure(db, missingSecretRun(overrides), { phase: "setup" });

    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it.each([undefined, { kind: "bootstrap" }, { kind: "bootstrap", providerWorkStarted: true }])(
    "reports missing-secret failures without explicit bootstrap proof: %j", async (executionRecovery) => {
      await seedCompanyAndAgent();
      const run = missingSecretRun();
      run.resultJson!.executionRecovery = executionRecovery;

      await reportRunFailure(db, run, { phase: "setup" });

      expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    },
  );

  it.each([undefined, "execute"] as const)("reports configuration failures outside setup: %s", async (phase) => {
    await seedCompanyAndAgent();

    await reportRunFailure(db, missingSecretRun(), { phase });

    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it("captures once for the status failed", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({ status: "failed" });

    await reportRunFailure(db, run);

    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  function restrictedWorkspaceSourceRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      errorCode: "configuration_incomplete",
      executionStage: "preparing",
      resultJson: {
        configurationIncomplete: { reason: "workspace_source_restricted", issueId: randomUUID() },
        executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
      },
      ...overrides,
    });
  }

  it("keeps a restricted shared workspace source actionable on the task", async () => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, restrictedWorkspaceSourceRun(), { phase: "setup" });
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it.each([
    { status: "timed_out" }, { errorCode: "setup_failed" }, { executionStage: "executing" }, { exitCode: 1 }, { signal: "SIGTERM" },
    { resultJson: { configurationIncomplete: { reason: "workspace_source_restricted" } } },
    { resultJson: { configurationIncomplete: { reason: "workspace_source_restricted" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } },
  ])("reports restricted-source markers without an exact pre-dispatch outcome: %j", async (overrides) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, restrictedWorkspaceSourceRun(overrides), { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it.each([undefined, "execute"] as const)("reports restricted-source markers outside the setup report: %s", async (phase) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, restrictedWorkspaceSourceRun(), { phase });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  function localPathConfigurationRun(overrides: Partial<typeof heartbeatRuns.$inferSelect> = {}) {
    return buildRun({
      errorCode: "workspace_validation_failed",
      executionStage: "preparing",
      resultJson: {
        workspaceValidation: {
          reason: "git_worktree_base_not_git_checkout",
          configurationReason: "local_path_requires_git_checkout",
          resolvedWorkspaceSource: "project_primary",
          workspaceStrategyType: "git_worktree",
          requestedExecutionWorkspaceMode: "isolated_workspace",
        },
        executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
      },
      ...overrides,
    });
  }

  it.each(["isolated_workspace", "operator_branch"])("keeps proven local-path policy conflicts actionable locally: %s", async (mode) => {
    await seedCompanyAndAgent();
    const run = localPathConfigurationRun();
    (run.resultJson!.workspaceValidation as Record<string, unknown>).requestedExecutionWorkspaceMode = mode;
    const before = structuredClone(run);

    await reportRunFailure(db, run, { phase: "setup" });

    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    expect(run).toEqual(before);
  });

  it.each([
    { configurationReason: undefined },
    { configurationReason: "unknown_reason" },
    { reason: "git_worktree_base_materialization_failed" },
    { reason: "git_worktree_not_reusable" },
    { reason: "inherited_workspace_reuse_unavailable" },
    { resolvedWorkspaceSource: "agent_home" },
    { workspaceStrategyType: "project_primary" },
    { requestedExecutionWorkspaceMode: "agent_default" },
  ])("reports unproven and other workspace failures: %j", async (validation) => {
    await seedCompanyAndAgent();
    const run = localPathConfigurationRun();
    Object.assign(run.resultJson!.workspaceValidation as object, validation);
    await reportRunFailure(db, run, { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it.each([
    { status: "timed_out" }, { errorCode: "setup_failed" }, { executionStage: "executing" },
    { exitCode: 1 }, { signal: "SIGTERM" }, { resultJson: {} },
    { resultJson: { workspaceValidation: { reason: "git_worktree_base_not_git_checkout", configurationReason: "local_path_requires_git_checkout" } } },
  ])("reports workspace failures without an exact pre-dispatch outcome: %j", async (overrides) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, localPathConfigurationRun(overrides), { phase: "setup" });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it.each([undefined, "execute"] as const)("reports local-path markers outside the setup report: %s", async (phase) => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, localPathConfigurationRun(), { phase });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
  });

  it.each([undefined, { kind: "bootstrap" }, { kind: "bootstrap", providerWorkStarted: true }])(
    "reports local-path markers without affirmative unstarted-provider evidence: %j", async (executionRecovery) => {
      await seedCompanyAndAgent();
      const run = localPathConfigurationRun();
      run.resultJson!.executionRecovery = executionRecovery;
      await reportRunFailure(db, run, { phase: "setup" });
      expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    },
  );

  it("captures once for the status timed_out", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({ status: "timed_out" });

    await reportRunFailure(db, run);

    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    expect(mockCaptureRunFailure).toHaveBeenCalledWith(
      expect.objectContaining({ runStatus: "timed_out" }),
    );
  });

  it("forwards stored exit evidence even when the adapter error is generic", async () => {
    await seedCompanyAndAgent();
    for (const processExit of [
      { exitCode: 1, signal: null },
      { exitCode: null, signal: "SIGTERM" },
      { exitCode: null, signal: null },
    ]) {
      mockCaptureRunFailure.mockClear();
      await reportRunFailure(db, buildRun({
        error: "Adapter failed",
        errorCode: "adapter_failed",
        ...processExit,
        stdoutExcerpt: "private-output",
        stderrExcerpt: "private-error-output",
        resultJson: { private: "adapter-result" },
      }));

      expect(mockCaptureRunFailure).toHaveBeenCalledWith(expect.objectContaining({
        errorMessage: "Adapter failed",
        errorCode: "adapter_failed",
        ...processExit,
      }));
      const captured = mockCaptureRunFailure.mock.calls[0][0];
      expect(captured).not.toHaveProperty("stdoutExcerpt");
      expect(captured).not.toHaveProperty("stderrExcerpt");
      expect(captured).not.toHaveProperty("resultJson");
    }
  });

  it("reports selected provider diagnostics and the original cause without copying arbitrary payloads", async () => {
    await seedCompanyAndAgent();
    const cause = Object.assign(new Error("provider unreachable"), {
      code: "ECONNRESET", requestId: "request-123", status: 503,
      response: { body: "private-response" },
    });
    const error = new Error("adapter threw", { cause });
    const run = buildRun({
      runtimeMode: "legacy",
      executionStage: "execute",
      error: "adapter threw",
      stderrExcerpt: "private-stderr",
      stdoutExcerpt: "private-stdout",
      contextSnapshot: { prompt: "private-prompt" },
      resultJson: {
        terminalSessionFailure: { category: "service", title: "Provider unavailable", details: "request-123 failed", raw: "private-provider-raw" },
        timeoutFired: false, summary: "private-summary", env: { SECRET: "private-env" },
      },
    });
    await reportRunFailure(db, run, { error, phase: "execute", adapterErrorMeta: { phase: "turn", retryable: true, response: "private-adapter-response" } });
    const captured = mockCaptureRunFailure.mock.calls[0][0];
    expect(captured.diagnostics).toMatchObject({
      execution: { runtimeMode: "legacy", executionStage: "execute", failurePhase: "execute", timeoutFired: false },
      adapter: { phase: "turn", retryable: true },
      provider: { category: "service", title: "Provider unavailable", details: "request-123 failed" },
      exceptions: [{ message: "adapter threw" }, { message: "provider unreachable", code: "ECONNRESET", status: 503, requestId: "request-123" }],
    });
    expect(captured.diagnostics.exceptions[0].stack).toContain("run-failure-report.test.ts");
    // Match the sensitive fixture values, not a legitimate checkout name in the stack.
    for (const value of ["private-response", "private-stderr", "private-stdout", "private-prompt", "private-provider-raw", "private-summary", "private-env", "private-adapter-response"]) {
      expect(JSON.stringify(captured)).not.toContain(value);
    }
    expect(error.cause).toBe(cause);
  });

  it.each([false, true])("redacts registered run secrets and fails closed when resolution fails: %s", async (fails) => {
    await seedCompanyAndAgent();
    const secret = "opaque-registered-value";
    const run = buildRun({
      error: `connection failed: ${secret}`,
      contextSnapshot: { paperclipSecretRedactions: [{ fingerprintSha256: "fixture", material: { encrypted: "fixture" } }] },
      resultJson: { terminalSessionFailure: { category: "service", details: `upstream rejected ${secret}` } },
    });
    await db.insert(heartbeatRuns).values(run);
    if (fails) mockResolveSecret.mockRejectedValueOnce(new Error("fixture resolution failed"));
    else mockResolveSecret.mockResolvedValueOnce(secret);

    await expect(reportRunFailure(db, run, {
      error: new Error(`failed ${secret}`, { cause: new Error(`cause ${secret}`) }),
      adapterErrorMeta: { causeMessage: `adapter ${secret}` },
    })).resolves.toBeUndefined();
    expect(mockResolveSecret).toHaveBeenCalledTimes(1);
    if (fails) {
      expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    } else {
      expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
      const captured = mockCaptureRunFailure.mock.calls[0][0];
      expect(JSON.stringify(captured)).not.toContain(secret);
      expect(captured.diagnostics.provider.details).toContain(REDACTED_EVENT_VALUE);
      expect(captured.diagnostics.exceptions[1].message).toContain(REDACTED_EVENT_VALUE);
    }
  });

  it("redacts runtime and host environment secret values even without a persisted registry", async () => {
    await seedCompanyAndAgent();
    const runtimeSecret = "opaque-runtime-value";
    const hostSecret = "opaque-host-value";
    vi.stubEnv("FIXTURE_CUSTOM_VALUE", hostSecret);
    const text = `connection failed: ${runtimeSecret} ${hostSecret}`;
    const run = buildRun({ error: text, contextSnapshot: null,
      resultJson: { terminalSessionFailure: { details: text } },
    });
    await reportRunFailure(db, run, {
      error: new Error(text, { cause: new Error(text) }),
      adapterErrorMeta: { causeMessage: text, stackPreview: text },
      secretValues: [runtimeSecret],
    });
    expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    const captured = mockCaptureRunFailure.mock.calls[0][0];
    expect(JSON.stringify(captured)).not.toContain(runtimeSecret);
    expect(JSON.stringify(captured)).not.toContain(hostSecret);
    expect(captured.diagnostics.exceptions).toHaveLength(2);
    expect(captured).not.toHaveProperty("secretValues");
  });

  it("still redacts short values from unknown host settings", async () => {
    await seedCompanyAndAgent();
    vi.stubEnv("FIXTURE_CUSTOM_VALUE", "1");

    await reportRunFailure(db, buildRun());

    expect(mockCaptureRunFailure.mock.calls[0][0].errorMessage)
      .toBe(`the provider process exited with code ${REDACTED_EVENT_VALUE}`);
  });

  it("waits for the terminal write when a native run is failed pending a same-run retry", async () => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, buildRun({ runtimeMode: "native", nativePhase: "retryable_failure" }));
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();

    await reportRunFailure(db, buildRun({ runtimeMode: "native", nativePhase: "terminal_failure" }));
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
  });

  it("treats leaving a pending native retry as a new terminal failure", () => {
    const pending = { status: "failed", runtimeMode: "native", nativePhase: "retryable_failure" } as const;
    expect(isNewTerminalRunFailure({ ...pending, status: "running", nativePhase: "observed" }, pending)).toBe(true);
    expect(isNewTerminalRunFailure(pending, { ...pending, nativePhase: "terminal_failure" })).toBe(true);
    expect(isNewTerminalRunFailure(pending, { ...pending, nativePhase: "committed" })).toBe(true);
    expect(isNewTerminalRunFailure(pending, pending)).toBe(false);
    expect(isNewTerminalRunFailure({ ...pending, nativePhase: null }, { ...pending, nativePhase: "terminal_failure" })).toBe(false);
    expect(isNewTerminalRunFailure({ ...pending, runtimeMode: "legacy" }, { ...pending, runtimeMode: "legacy", nativePhase: null }))
      .toBe(false);
  });

  it("captures nothing for succeeded, cancelled, and interrupted", async () => {
    await seedCompanyAndAgent();
    for (const status of ["succeeded", "cancelled", "interrupted"] as const) {
      const run = buildRun({ status });
      await reportRunFailure(db, run);
    }

    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });
  it.each(["provider", "unknown"])("reports an unexpected started cancellation from %s", async source => {
    await seedCompanyAndAgent();
    const run = buildRun({ status: "cancelled", startedAt: new Date(0), finishedAt: new Date(1000),
      resultJson: { cancellation: { source, expected: false, initiator: { type: "provider", id: "private-actor" },
        reason: "private-reason", recordedAt: new Date(1000).toISOString() } } });
    await reportRunFailure(db, run);
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
    expect(mockCaptureRunFailure.mock.calls[0][0]).toMatchObject({ runStatus: "cancelled",
      diagnostics: { execution: { cancellationSource: source, cancellationExpected: false } } });
    expect(JSON.stringify(mockCaptureRunFailure.mock.calls)).not.toMatch(/private-actor|private-reason/);
  });
  it("labels a host controller lease loss with its closed cause", async () => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, buildRun({ status: "cancelled", startedAt: new Date(0), finishedAt: new Date(1000),
      resultJson: { cancellation: controllerLeaseLossCancellation("renewal_deadline") } }));
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
    expect(mockCaptureRunFailure.mock.calls[0][0]).toMatchObject({ diagnostics: { execution: {
      cancellationSource: "controller_lease", cancellationExpected: false, controllerLeaseLossCause: "renewal_deadline" } } });
    expect(JSON.stringify(mockCaptureRunFailure.mock.calls)).not.toMatch(/Legacy controller lease lost/);
  });
  it("reports bounded lease-loss timings for a setup failure", async () => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, buildRun({ status: "failed", errorCode: "setup_failed", startedAt: new Date(0),
      finishedAt: new Date(1000), resultJson: { controllerLeaseLoss: { cause: "renewal_deadline",
        sinceRenewalMs: 61_000.4, renewalPendingMs: 59_000, deadlineLateMs: 1e12, renewalCount: -1, note: "private-note" } } }));
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
    const execution = mockCaptureRunFailure.mock.calls[0][0].diagnostics.execution;
    expect(execution).toMatchObject({ controllerLeaseLossCause: "renewal_deadline", controllerLeaseSinceRenewalMs: 61_000,
      controllerLeaseRenewalPendingMs: 59_000, controllerLeaseDeadlineLateMs: 86_400_000 });
    expect(execution).not.toHaveProperty("controllerLeaseRenewalCount");
    expect(JSON.stringify(mockCaptureRunFailure.mock.calls)).not.toMatch(/private-note/);
  });
  it("drops lease-loss timings without a closed cause", async () => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, buildRun({ status: "failed", errorCode: "setup_failed", startedAt: new Date(0),
      finishedAt: new Date(1000), resultJson: { controllerLeaseLoss: { cause: "forged", sinceRenewalMs: 1 } } }));
    expect(mockCaptureRunFailure).toHaveBeenCalledOnce();
    const execution = mockCaptureRunFailure.mock.calls[0][0].diagnostics.execution;
    expect(execution).not.toHaveProperty("controllerLeaseLossCause");
    expect(execution).not.toHaveProperty("controllerLeaseSinceRenewalMs");
  });
  it("does not report an operator's Stop as a failure", async () => {
    await seedCompanyAndAgent();
    await reportRunFailure(db, buildRun({ status: "cancelled", startedAt: new Date(0), resultJson: {
      cancellation: { source: "operator", expected: true, initiator: { type: "user", id: "board" },
        reason: "Stop", recordedAt: new Date().toISOString() },
    } }));
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  it("sends agents.adapterType from the loaded agent row", async () => {
    await seedCompanyAndAgent({ adapterType: "claude_managed" });
    const run = buildRun({ status: "failed" });

    await reportRunFailure(db, run);

    expect(mockCaptureRunFailure).toHaveBeenCalledWith(
      expect.objectContaining({ agentAdapter: "claude_managed" }),
    );
  });

  it("sends the adapter value unknown when the agent row is absent", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({ status: "failed", agentId: randomUUID() });

    await reportRunFailure(db, run);

    expect(mockCaptureRunFailure).toHaveBeenCalledWith(
      expect.objectContaining({ agentAdapter: "unknown" }),
    );
  });

  it("calls redactCurrentUserText on the error message before it composes the credential redactor", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({ status: "failed", error: "raw message" });

    await reportRunFailure(db, run);

    expect(mockRedactCurrentUserText).toHaveBeenCalledWith("raw message");
  });

  it("removes an Authorization: Bearer credential from the error message", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({
      status: "failed",
      error: "the adapter request failed: Authorization: Bearer live-secret-token-value",
    });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorMessage).not.toContain("live-secret-token-value");
    expect(errorMessage).toContain(REDACTED_EVENT_VALUE);
  });

  it("removes an API-key form from the error message", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({
      status: "failed",
      error: `adapter payload: {"apiKey":"json-secret-value"}`,
    });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorMessage).not.toContain("json-secret-value");
    expect(errorMessage).toContain(REDACTED_EVENT_VALUE);
  });

  it("removes a JSON Web Token form from the error message", async () => {
    await seedCompanyAndAgent();
    const jwt =
      "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
    const run = buildRun({ status: "failed", error: `session token: ${jwt}` });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorMessage).not.toContain(jwt);
    expect(errorMessage).toContain(REDACTED_EVENT_VALUE);
  });

  it("removes a password value from the error message", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({
      status: "failed",
      error: `login failed: password="hunter2-super-secret"`,
    });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorMessage).not.toContain("hunter2-super-secret");
    expect(errorMessage).toContain(REDACTED_EVENT_VALUE);
  });

  it("removes a database connection string from the error message", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({
      status: "failed",
      error: `connect failed: connectionString: "postgres://appuser:[email protected]:5432/paperclip"`,
    });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorMessage).not.toContain("s3cr3t-pass");
    expect(errorMessage).toContain(REDACTED_EVENT_VALUE);
  });

  it("still masks the current user's home path in the error message", async () => {
    await seedCompanyAndAgent();
    const homeDir = os.homedir();
    const rawError = `read failed at ${homeDir}/workspace/report.log`;
    const run = buildRun({ status: "failed", error: rawError });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    // Environment-value redaction can fully mask the username before the
    // current-user redactor applies its partial mask. Both remove the home path.
    expect(errorMessage).toContain("workspace/report.log");
    expect(errorMessage).not.toContain(homeDir);
  });

  const MAX_ERROR_MESSAGE_LENGTH = 4096;

  it("truncates an error message that is longer than the bound", async () => {
    await seedCompanyAndAgent();
    const longError = "x".repeat(MAX_ERROR_MESSAGE_LENGTH + 500);
    const run = buildRun({ status: "failed", error: longError });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorMessage).toHaveLength(MAX_ERROR_MESSAGE_LENGTH);
    expect(errorMessage).toBe("x".repeat(MAX_ERROR_MESSAGE_LENGTH - 12) + "\n[truncated]");
  });

  it("preserves a short error message with known public host settings", async () => {
    vi.stubEnv("PAPERCLIP_DB_BACKUP_ENABLED", "false");
    vi.stubEnv("PAPERCLIP_DB_BACKUP_RETENTION_DAYS", "1");
    vi.stubEnv("GITHUB_RUN_ATTEMPT", "1");
    await seedCompanyAndAgent();
    const shortError = "the provider process exited with code 1";
    const run = buildRun({ status: "failed", error: shortError });

    await reportRunFailure(db, run);

    const { errorMessage } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorMessage).toBe(shortError);
  });

  const MAX_ERROR_CODE_LENGTH = 200;

  it("sends a normal error code such as adapter_failed to Sentry unchanged", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({ status: "failed", errorCode: "adapter_failed" });

    await reportRunFailure(db, run);

    expect(mockCaptureRunFailure).toHaveBeenCalledWith(
      expect.objectContaining({ errorCode: "adapter_failed" }),
    );
  });

  it("redacts an error code that holds a credential form", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({
      status: "failed",
      errorCode: "Authorization: Bearer live-secret-token-value",
    });

    await reportRunFailure(db, run);

    const { errorCode } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorCode).not.toContain("live-secret-token-value");
    expect(errorCode).toContain(REDACTED_EVENT_VALUE);
  });

  it("truncates an error code that is longer than 200 characters", async () => {
    await seedCompanyAndAgent();
    const longErrorCode = "y".repeat(MAX_ERROR_CODE_LENGTH + 50);
    const run = buildRun({ status: "failed", errorCode: longErrorCode });

    await reportRunFailure(db, run);

    const { errorCode } = mockCaptureRunFailure.mock.calls[0][0];
    expect(errorCode).toHaveLength(MAX_ERROR_CODE_LENGTH);
    expect(errorCode).toBe("y".repeat(MAX_ERROR_CODE_LENGTH - 12) + "\n[truncated]");
  });

  it("sends errorCode null unchanged when the run holds no error code", async () => {
    await seedCompanyAndAgent();
    const run = buildRun({ status: "failed", errorCode: null });

    await reportRunFailure(db, run);

    expect(mockCaptureRunFailure).toHaveBeenCalledWith(
      expect.objectContaining({ errorCode: null }),
    );
  });

  it("reads the task id from nativeIssueId, falling back to contextSnapshot.issueId", async () => {
    await seedCompanyAndAgent();
    const nativeIssueId = randomUUID();
    const runWithNativeIssueId = buildRun({
      status: "failed",
      nativeIssueId,
      contextSnapshot: { issueId: randomUUID() },
    });

    await reportRunFailure(db, runWithNativeIssueId);

    expect(mockCaptureRunFailure).toHaveBeenCalledWith(
      expect.objectContaining({ taskId: nativeIssueId }),
    );

    mockCaptureRunFailure.mockClear();
    const contextIssueId = randomUUID();
    const runWithContextIssueId = buildRun({
      status: "failed",
      nativeIssueId: null,
      contextSnapshot: { issueId: contextIssueId },
    });

    await reportRunFailure(db, runWithContextIssueId);

    expect(mockCaptureRunFailure).toHaveBeenCalledWith(
      expect.objectContaining({ taskId: contextIssueId }),
    );
  });

  it("does not throw when the database read fails", async () => {
    const throwingDb = {
      select: () => {
        throw new Error("connection reset");
      },
    } as unknown as Db;
    const run = buildRun({ status: "failed", agentId: randomUUID() });

    await expect(reportRunFailure(throwingDb, run)).resolves.toBeUndefined();
    expect(mockCaptureRunFailure).not.toHaveBeenCalled();
  });

  describe("waitForPendingRunFailureReports", () => {
    function deferredAgentRows() {
      let resolve!: (rows: Array<{ adapterType: string }>) => void;
      const promise = new Promise<Array<{ adapterType: string }>>((res) => {
        resolve = res;
      });
      return { promise, resolve };
    }

    function fakeDb(agentRowsPromise: Promise<Array<{ adapterType: string }>>): Db {
      return {
        select: () => ({
          from: () => ({
            where: () => agentRowsPromise,
          }),
        }),
      } as unknown as Db;
    }

    it("resolves at once when no report is in flight", async () => {
      await expect(waitForPendingRunFailureReports()).resolves.toBeUndefined();
    });

    it("waits for an unawaited report's database read and Sentry capture before it resolves", async () => {
      const agentRows = deferredAgentRows();
      const run = buildRun({ status: "failed", agentId: randomUUID() });

      // Do not await the report — this models the fire-and-forget
      // `void reportRunFailure(db, run)` call every caller uses.
      void reportRunFailure(fakeDb(agentRows.promise), run);

      const drain = waitForPendingRunFailureReports(1_000);
      await new Promise((resolve) => setTimeout(resolve, 10));
      expect(mockCaptureRunFailure).not.toHaveBeenCalled();

      agentRows.resolve([{ adapterType: "claude_managed" }]);
      await drain;

      expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1);
    });

    it("gives up after the bound and does not throw when a report never settles", async () => {
      const run = buildRun({ status: "failed", agentId: randomUUID() });
      void reportRunFailure(fakeDb(new Promise(() => undefined)), run);

      await expect(waitForPendingRunFailureReports(20)).resolves.toBeUndefined();
      expect(mockCaptureRunFailure).not.toHaveBeenCalled();
    });
  });
});