The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
import { randomUUID } from "node:crypto";
import type { Db } from "@paperclipai/db";
import { agents, agentWakeupRequests, caseHandoffs, cases, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
import { eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import * as usageReceipts from "../usage-receipts.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { UnresolvedWorkspaceBaseRefError } from "../workspace-runtime.js";
import { LegacyControllerLeaseLostError } from "../legacy-controller-lease.js";
import { requestedRunCancellation } from "../run-cancellation.js";
import { ProjectAccessDeniedError } from "../project-access-denial.js";
import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js";
import { createHeartbeatRunPreparation, ConfigurationIncompleteFailure } from "./run-preparation.js";
import { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js";
import { buildEffectiveRunSessionConfigMetadata } from "./workspaces.js";
import { createHeartbeatRunCompletion, type CompleteHeartbeatRunInput, type FailHeartbeatRunInput, type HeartbeatRunCompletionDependencies } from "./run-completion.js";
import { createHeartbeatAgentStatus } from "./agent-status.js";
vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
type Dependencies = HeartbeatRunCompletionDependencies;
function callbacks(db: Db) {
const state = createHeartbeatRunState(db);
const lifecycleDeps = {
...state, ...createHeartbeatRunPreparation(db),
issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() },
recovery: { escalateStrandedAssignedIssue: vi.fn() },
companySkills: { completeTestRunForIssue: vi.fn() },
budgets: { getInvocationBlock: vi.fn(async () => null) },
treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },
enqueueWakeup: vi.fn(),
getCurrentUserRedactionOptions: vi.fn(async () => ({ enabled: false })),
getAgentInvokability: vi.fn<HeartbeatLifecycleDependencies["getAgentInvokability"]>(async () => ({ invokable: true })),
emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {},
} satisfies HeartbeatLifecycleDependencies;
const lifecycle = createHeartbeatLifecycle(db, lifecycleDeps);
return {
...state, ...lifecycle,
setRunStatusIfRunning: vi.fn(lifecycle.setRunStatusIfRunning),
setWakeupStatus: vi.fn(lifecycle.setWakeupStatus),
upsertTaskSession: vi.fn(state.upsertTaskSession),
clearTaskSessions: vi.fn(state.clearTaskSessions),
classifyAndPersistRunLiveness: vi.fn<Dependencies["classifyAndPersistRunLiveness"]>(async run => run),
completeSkillTestRunForHeartbeatOutcome: vi.fn<Dependencies["completeSkillTestRunForHeartbeatOutcome"]>(async () => null),
refreshContinuationSummaryForRun: vi.fn<Dependencies["refreshContinuationSummaryForRun"]>(async () => null),
findRunIssueComment: vi.fn<Dependencies["findRunIssueComment"]>(async () => null),
findLatestCompletedFinalAgentMessage: vi.fn<Dependencies["findLatestCompletedFinalAgentMessage"]>(async () => null),
finalizeIssueCommentPolicy: vi.fn<Dependencies["finalizeIssueCommentPolicy"]>(async () => ({ outcome: "not_applicable", queuedRun: null })),
handleIssueReviewPathDisposition: vi.fn<Dependencies["handleIssueReviewPathDisposition"]>(async () => undefined),
handleRunLivenessContinuation: vi.fn<Dependencies["handleRunLivenessContinuation"]>(),
handleSuccessfulRunHandoff: vi.fn<Dependencies["handleSuccessfulRunHandoff"]>(),
updateRuntimeState: vi.fn<Dependencies["updateRuntimeState"]>(),
scheduleBoundedRetryForRun: vi.fn<Dependencies["scheduleBoundedRetryForRun"]>(),
scheduleInteractionContinuationInfrastructureRetryIfEligible: vi.fn<Dependencies["scheduleInteractionContinuationInfrastructureRetryIfEligible"]>(async () => null),
timerClaimWasFirstHeartbeat: vi.fn<Dependencies["timerClaimWasFirstHeartbeat"]>(() => undefined),
releaseIssueExecutionAndPromote: vi.fn<Dependencies["releaseIssueExecutionAndPromote"]>(async () => undefined),
finalizeAgentStatus: vi.fn<Dependencies["finalizeAgentStatus"]>(async () => undefined),
enqueueWakeup: vi.fn<Dependencies["enqueueWakeup"]>(),
issuesSvc: { addComment: vi.fn<Dependencies["issuesSvc"]["addComment"]>() },
recovery: { reconcileLegacyContinuation: vi.fn<Dependencies["recovery"]["reconcileLegacyContinuation"]>(), reconcileResolvedDependencyWakeBackstop: vi.fn<Dependencies["recovery"]["reconcileResolvedDependencyWakeBackstop"]>() },
getCurrentUserRedactionOptions: lifecycleDeps.getCurrentUserRedactionOptions,
budgetHooks: {},
runLogStore: { finalize: vi.fn<Dependencies["runLogStore"]["finalize"]>(async () => ({ bytes: 123, sha256: "log-hash", compressed: true })) },
traceStore: { finalize: vi.fn<Dependencies["traceStore"]["finalize"]>() },
processRunCancellationSettlements: new Map<string, { settled: Promise<void>; failed: boolean }>(),
failedProcessRunCancellations: new Map<string, { settled: Promise<void>; failed: boolean }>(),
} satisfies Dependencies;
}
it("constructs completion handlers without querying or starting work", () => {
const access = vi.fn(() => { throw new Error("Unexpected database access"); });
const db = new Proxy({}, { get: access }) as Db;
const deps = callbacks(db);
createHeartbeatRunCompletion(db, deps);
expect(access).not.toHaveBeenCalled();
for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled();
});
const support = await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)("heartbeat run completion boundary", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: Db;
beforeAll(async () => {
// Keep the accounting transaction real without reading provider receipt files.
vi.spyOn(usageReceipts, "indexPendingUsageReceipts").mockResolvedValue(new Map());
vi.spyOn(usageReceipts, "recoverPendingRunUsageReceipts").mockResolvedValue([]);
const external = process.env.PAPERCLIP_TEST_DATABASE_URL;
if (external) db = createDb(external);
else {
database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-completion");
db = createDb(database.connectionString);
}
await db.execute(sql`set client_min_messages = warning`);
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); vi.restoreAllMocks(); });
beforeEach(async () => { await db.execute(sql`truncate table companies restart identity cascade`); });
async function fixture(patch: Partial<typeof heartbeatRuns.$inferInsert> = {}) {
const [company] = await db.insert(companies).values({ name: "Completion", issuePrefix: "CMP" }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Completing agent", adapterType: "process" }).returning();
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "on_demand", status: "claimed" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, wakeupRequestId: wake.id, status: "running", runtimeMode: "legacy", ...patch }).returning();
const deps = callbacks(db);
const controller = new AbortController();
const input: CompleteHeartbeatRunInput = {
run, agent, issueId: null, issueRef: null, signal: controller.signal,
output: { handle: { store: "local_file", logRef: "test-log" }, outputProgressState: { pending: { bytes: 7 } }, flushOutputProgress: vi.fn(async () => {}), stdoutExcerpt: "answer", stderrExcerpt: "diagnostic" },
session: { runtimeForAdapter: { sessionId: null, sessionDisplayId: null }, previousSessionParams: null, taskKey: "completion-task", configuredModel: "test-model",
sessionConfigMetadata: await buildEffectiveRunSessionConfigMetadata({ adapterType: agent.adapterType, effectiveAdapterConfig: {}, agentRuntimeConfig: {}, issueOverrides: null, workspaceConfig: null, environment: null, environmentEnv: null, projectEnv: null, routineEnv: null, runtimeSkills: [] }),
},
readFailureReportSecrets: () => [],
adapterResult: { exitCode: 0, signal: null, timedOut: false, sessionId: "provider-session", sessionParams: { sessionId: "provider-session" }, usage: { inputTokens: 20, cachedInputTokens: 5, outputTokens: 8 }, usageBasis: "per_run", costUsd: 0.01, provider: "test", model: "test-model", resultJson: { finalResponse: "Completed the work." } },
sessionCodec: getAdapterSessionCodec(agent.adapterType), taskSessionForRun: null, sessionCompaction: { rotate: false, reason: null }, configFreshnessResultMetadata: { version: 1 }, runLedgerScope: {}, currentUserRedactionOptions: { enabled: false }, providerTraceCapture: true, onProviderTraceFinalized: vi.fn(), issueContext: null, onLog: vi.fn(async () => {}),
};
const completion = createHeartbeatRunCompletion(db, deps);
const failure = (err: unknown): FailHeartbeatRunInput => ({ ...input, runId: run.id, err, identityRedactor: createAgentIdentityRedactor("private-identity-material"), requiredWorkspaceRestoreEvidence: { workspaceRestoreFailure: { reason: "missing_base" } }, legacyAdapterEntered: true, goalCheckpointSession: { current: null }, previousSessionDisplayId: "previous-session", taskSession: null });
const setup = (outerErr: unknown, providerDispatchStarted = false) => ({ run, runId: run.id, outerErr, signal: controller.signal, providerDispatchStarted, identityRedactor: createAgentIdentityRedactor(), readFailureReportSecrets: () => [] });
return { run, agent, wake, deps, controller, input, completion, failure, setup };
}
const read = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!);
it.each(["completeRun", "failRun"] as const)("settles the outgoing agent after a case handoff in %s", async operation => {
const f = await fixture();
await db.update(agents).set({ status: "running", lifecycleState: "ready" }).where(eq(agents.id, f.agent.id));
const [recipient] = await db.insert(agents).values({ companyId: f.agent.companyId, name: "Next phase", adapterType: "process" }).returning();
const [issue] = await db.insert(issues).values({ companyId: f.agent.companyId, title: "Case task", status: "in_progress", assigneeAgentId: f.agent.id }).returning();
const [record] = await db.insert(cases).values({ companyId: f.agent.companyId, caseNumber: 1, identifier: "CASE-1", caseType: "work", title: issue.title, workIssueId: issue.id, handoffVersion: 1 }).returning();
await db.insert(caseHandoffs).values({ companyId: f.agent.companyId, caseId: record.id, issueId: issue.id, version: 1,
fromAgentId: f.agent.id, toAgentId: recipient.id, phase: "verification", instructions: "Verify the saved work",
actorType: "agent", actorId: f.agent.id, sourceRunId: f.run.id, wakeupRequestId: randomUUID(), idempotencyKey: "next-phase", inputFingerprint: "next-phase", state: "awaiting_release" });
f.input.issueId = issue.id;
f.input.issueRef = issue;
f.deps.finalizeAgentStatus.mockImplementation(createHeartbeatAgentStatus(db).finalizeAgentStatus);
if (operation === "completeRun") await f.completion.completeRun(f.input);
else await f.completion.failRun(f.failure(new Error("Provider exited after the accepted handoff")));
const [settledAgent] = await db.select().from(agents).where(eq(agents.id, f.agent.id));
expect(settledAgent).toMatchObject({ status: operation === "completeRun" ? "idle" : "error", lastHeartbeatAt: expect.any(Date) });
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), { suppressImmediateRecovery: true });
expect(f.deps.handleSuccessfulRunHandoff).not.toHaveBeenCalled();
expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0]?.status).toBe("in_progress");
});
it("persists usage, accounting, final logs, presentation and the next task session", async () => {
const f = await fixture();
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", sessionIdAfter: "provider-session", logBytes: 123, logSha256: "log-hash", logCompressed: true, stdoutExcerpt: "answer", costAccountingPending: false, costAccountedAt: expect.any(Date), usageJson: { inputTokens: 20, cachedInputTokens: 5, outputTokens: 8, usageSource: "per_run", configFreshness: { version: 1 } }, resultJson: { presentationDecision: { commentAction: "none" } } });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(1);
expect((await f.deps.getTaskSession(f.agent.companyId, f.agent.id, f.agent.adapterType, "completion-task"))?.sessionDisplayId).toBe("provider-session");
expect(f.input.output.outputProgressState.pending?.bytes).toBe(123);
expect(f.input.onProviderTraceFinalized).toHaveBeenCalledOnce();
expect(f.deps.setWakeupStatus).toHaveBeenCalledWith(f.wake.id, "completed", expect.any(Object));
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object));
});
it.each([
{ timedOut: true, expected: "timed_out", errorCode: "timeout" },
{ signal: "SIGTERM", expected: "failed", errorCode: "adapter_failed" },
{ errorMessage: "provider quota", errorCode: "provider_quota", expected: "failed" },
])("keeps adapter outcome classification ($expected, $errorCode)", async ({ expected, ...result }) => {
const f = await fixture();
Object.assign(f.input.adapterResult, result);
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: expected, errorCode: result.errorCode });
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, expected, expect.any(String), expect.objectContaining({ keepIdleOnFailure: result.errorCode === "provider_quota" }));
});
it.each(["completeRun", "failRun"] as const)("waits for an owned Stop before %s can write a competing outcome", async operation => {
const f = await fixture();
let settle!: () => void;
const settled = new Promise<void>(resolve => { settle = resolve; });
f.deps.processRunCancellationSettlements.set(f.run.id, { settled, failed: false });
const completion = operation === "completeRun" ? f.completion.completeRun(f.input) : f.completion.failRun(f.failure(new Error("late adapter error")));
await Promise.resolve();
expect(f.deps.setRunStatusIfRunning).not.toHaveBeenCalled();
await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
f.controller.abort();
settle();
await completion;
expect((await read(f.run.id)).status).toBe("cancelled");
if (operation === "completeRun") {
expect((await read(f.run.id)).sessionIdAfter).toBe("provider-session");
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "cancelled", expect.any(String), expect.any(Object));
} else expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();
});
it("attributes an adapter stop caused by a lost controller lease", async () => {
const f = await fixture();
Object.assign(f.input.adapterResult, { exitCode: null, errorCode: "cancelled", errorMessage: "Grok execution was cancelled" });
f.controller.abort(new LegacyControllerLeaseLostError("renewal_deadline"));
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: "cancelled", resultJson: { cancellation: {
source: "controller_lease", expected: false, initiator: { type: "system" }, reason: "Legacy controller lease lost (renewal_deadline)" } } });
});
it("attributes a lost controller lease when the adapter throws the abort reason", async () => {
const f = await fixture();
f.controller.abort(new LegacyControllerLeaseLostError("renewal_rejected"));
await f.completion.failRun(f.failure(f.controller.signal.reason));
expect(await read(f.run.id)).toMatchObject({ status: "cancelled", resultJson: {
workspaceRestoreFailure: { reason: "missing_base" },
cancellation: { source: "controller_lease", expected: false, reason: "Legacy controller lease lost (renewal_rejected)" } } });
});
it("records the watchdog timings with a lease loss that stops the adapter", async () => {
const f = await fixture();
Object.assign(f.input.adapterResult, { exitCode: null, errorCode: "cancelled", errorMessage: "Grok execution was cancelled" });
f.controller.abort(new LegacyControllerLeaseLostError("renewal_deadline",
{ sinceRenewalMs: 61_000, renewalPendingMs: 58_000, deadlineLateMs: 2, renewalCount: 41 }));
await f.completion.completeRun(f.input);
expect((await read(f.run.id)).resultJson).toMatchObject({ controllerLeaseLoss: {
cause: "renewal_deadline", sinceRenewalMs: 61_000, renewalPendingMs: 58_000, deadlineLateMs: 2, renewalCount: 41 } });
});
it("records the lease-loss cause and timings when setup fails on the abort reason", async () => {
const f = await fixture();
f.controller.abort(new LegacyControllerLeaseLostError("renewal_deadline",
{ sinceRenewalMs: 60_004, renewalPendingMs: null, deadlineLateMs: 55_000, renewalCount: 3 }));
await f.completion.failRunSetup(f.setup(f.controller.signal.reason));
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "controller_lease_lost", resultJson: {
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
controllerLeaseLoss: { cause: "renewal_deadline", sinceRenewalMs: 60_004, renewalPendingMs: null,
deadlineLateMs: 55_000, renewalCount: 3 } } });
});
it("keeps a stop requested before the lease loss when the adapter throws", async () => {
const f = await fixture();
const requested = requestedRunCancellation({ cancelledByActorType: "user", cancelledByUserId: "operator-1" }, "Stopped from the board");
await db.update(heartbeatRuns).set({ resultJson: { cancellation: requested } }).where(eq(heartbeatRuns.id, f.run.id));
f.controller.abort(new LegacyControllerLeaseLostError("renewal_error"));
await f.completion.failRun(f.failure(f.controller.signal.reason));
expect(await read(f.run.id)).toMatchObject({ status: "cancelled", resultJson: { cancellation: {
source: "operator", expected: true, initiator: { type: "user", id: "operator-1" }, reason: "Stopped from the board" } } });
});
it("does not publish or overwrite a conflicting terminal transition that wins the compare-and-set", async () => {
const f = await fixture();
const write = f.deps.setRunStatusIfRunning.getMockImplementation()!;
f.deps.setRunStatusIfRunning.mockImplementationOnce(async (...args) => {
await db.update(heartbeatRuns).set({ status: "cancelled", errorCode: "cancelled", resultJson: { winner: "Stop" } }).where(eq(heartbeatRuns.id, f.run.id));
return write(...args);
});
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: "cancelled", resultJson: { winner: "Stop" }, sessionIdAfter: null });
expect(f.deps.setWakeupStatus).not.toHaveBeenCalled();
expect(f.deps.updateRuntimeState).not.toHaveBeenCalled();
expect(f.deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled();
expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();
});
it("retains native completion metadata when reconciliation already committed the same terminal status", async () => {
const finishedAt = new Date("2026-10-09T12:00:00Z");
const f = await fixture({ runtimeMode: "native", status: "succeeded", finishedAt, resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native" } });
f.input.adapterResult.nativeFinalization = { schema: "paperclip.native-finalization.v1", runtimeMode: "native", runId: f.run.id, companyId: f.run.companyId, issueId: "unused", result: {}, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done" }, turnId: null, sourceInstanceId: "test", normalizedSessionId: "native-session", providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, workspaceFinalizeStatus: "succeeded" };
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", finishedAt, sessionIdAfter: "provider-session", resultJson: { durable: "accepted" }, usageJson: { durableReceipt: "native", inputTokens: 20 } });
expect(f.deps.updateRuntimeState).toHaveBeenCalledOnce();
});
it("reports trace completion before a later write throws and reads output after log finalization", async () => {
const f = await fixture();
let excerpt = "before finalize";
Object.defineProperty(f.input.output, "stdoutExcerpt", { get: () => excerpt });
f.deps.runLogStore.finalize.mockImplementationOnce(async () => { excerpt = "last provider output"; return { bytes: 99, compressed: false }; });
const writeError = new Error("write failed");
f.deps.setRunStatusIfRunning.mockRejectedValueOnce(writeError);
await expect(f.completion.completeRun(f.input)).rejects.toBe(writeError);
expect(f.input.onProviderTraceFinalized).toHaveBeenCalledOnce();
expect(f.deps.setRunStatusIfRunning).toHaveBeenCalledWith(f.run.id, "succeeded", expect.objectContaining({ stdoutExcerpt: "last provider output", logBytes: 99 }), expect.any(Object));
});
it("keeps a failed trace finalization eligible for the executor's cleanup retry", async () => {
const f = await fixture();
f.deps.traceStore.finalize.mockRejectedValueOnce(new Error("trace store unavailable"));
await f.completion.completeRun(f.input);
expect((await read(f.run.id)).status).toBe("succeeded");
expect(f.input.onProviderTraceFinalized).not.toHaveBeenCalled();
});
it("keeps completion and session persistence moving after a response comment fails", async () => {
const f = await fixture();
f.input.adapterResult.resultJson = { finalResponseDisposition: "final", finalResponse: "Completed the work." };
const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Report the result" }).returning();
f.input.issueId = issue.id;
f.input.issueRef = issue;
f.input.issueContext = await createHeartbeatRunPreparation(db).getIssueExecutionContext(f.run.companyId, issue.id);
f.deps.issuesSvc.addComment.mockRejectedValueOnce(new Error("comment unavailable"));
await f.completion.completeRun(f.input);
expect(f.deps.issuesSvc.addComment).toHaveBeenCalledOnce();
expect(f.input.onLog).toHaveBeenCalledWith("stderr", expect.stringContaining("comment unavailable"));
expect((await read(f.run.id)).status).toBe("succeeded");
expect(f.deps.upsertTaskSession).toHaveBeenCalledOnce();
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
});
it("clears only the completed task session when the adapter requests a reset", async () => {
const f = await fixture();
f.input.adapterResult.clearSession = true;
await f.completion.completeRun(f.input);
expect(f.deps.clearTaskSessions).toHaveBeenCalledWith(f.run.companyId, f.agent.id, { taskKey: "completion-task", adapterType: "process", expectedRunId: f.run.id });
expect(f.deps.upsertTaskSession).not.toHaveBeenCalled();
});
it("redacts execution failures, retains restore evidence and resumes the latest checkpoint", async () => {
const f = await fixture();
const failure = f.failure(new Error("private-identity-material crashed"));
failure.goalCheckpointSession.current = { params: { sessionId: "checkpoint" }, displayId: "checkpoint" };
await f.completion.failRun(failure);
const run = await read(f.run.id);
expect(run).toMatchObject({ status: "failed", errorCode: "adapter_failed", logBytes: 123, resultJson: { workspaceRestoreFailure: { reason: "missing_base" } } });
expect(run.error).not.toContain("private-identity-material");
expect((await f.deps.getTaskSession(f.agent.companyId, f.agent.id, f.agent.adapterType, "completion-task"))?.sessionDisplayId).toBe("checkpoint");
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
});
it("still records failure and releases issue ownership when log finalization and progress flushing fail", async () => {
const f = await fixture();
f.deps.runLogStore.finalize.mockRejectedValueOnce(new Error("log unavailable"));
f.input.output.flushOutputProgress = vi.fn(async () => { throw new Error("progress unavailable"); });
await f.completion.failRun(f.failure(new Error("adapter failed")));
expect((await read(f.run.id)).status).toBe("failed");
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
});
it("keeps durable result-less native failure authoritative and suppresses generic replacement recovery", async () => {
const f = await fixture({ runtimeMode: "native", status: "failed", errorCode: "native_result_missing" });
const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Native recovery" }).returning();
await db.update(heartbeatRuns).set({ nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
await db.insert(nativeRunFinalizations).values({ runId: f.run.id, companyId: f.run.companyId, issueId: issue.id, phase: "terminal_failure", failureCode: "native_result_missing" });
await f.completion.failRun(f.failure(new Error("transport ended")));
expect((await read(f.run.id)).errorCode).toBe("native_result_missing");
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), { suppressImmediateRecovery: true });
});
it("persists a project denial without provider work or an automatic retry", async () => {
const f = await fixture();
const projectId = randomUUID();
await f.completion.failRunSetup(f.setup(new ProjectAccessDeniedError(projectId)));
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "project_access_denied",
resultJson: { projectAccessDenied: { projectId }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });
expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled();
expect(f.deps.enqueueWakeup).not.toHaveBeenCalled();
expect(f.deps.scheduleInteractionContinuationInfrastructureRetryIfEligible).not.toHaveBeenCalled();
expect(f.deps.finalizeIssueCommentPolicy).not.toHaveBeenCalled();
});
it("records a setup configuration gap without inventing provider work or applying comment policy", async () => {
const f = await fixture();
await f.completion.failRunSetup(f.setup(new ConfigurationIncompleteFailure("missing credential", { configurationIncomplete: { reason: "missing_binding" } })));
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: { configurationIncomplete: { reason: "missing_binding" }, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });
expect(f.deps.finalizeIssueCommentPolicy).not.toHaveBeenCalled();
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce();
});
it("retains the default branch needed to repair an unresolved workspace base ref", async () => {
const f = await fixture();
const error = new UnresolvedWorkspaceBaseRefError({
requestedRef: "main", recoveryIdentityRef: "origin/main", attemptedRefs: ["origin/main"], defaultBranch: "master",
});
await f.completion.failRunSetup(f.setup(error));
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "configuration_incomplete", resultJson: {
configurationIncomplete: { reason: "workspace_base_ref_unresolved", requestedRef: "main", defaultBranch: "master", attemptedRefs: ["origin/main"], fingerprint: "workspace_base_ref:origin/main" },
executionRecovery: { kind: "bootstrap", providerWorkStarted: false },
} });
});
it("records a lease lost before dispatch as a retryable host stop, not an opaque setup failure", async () => {
const f = await fixture();
f.controller.abort(new LegacyControllerLeaseLostError("renewal_rejected"));
await f.completion.failRunSetup(f.setup(f.controller.signal.reason));
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "controller_lease_lost", resultJson: {
cancellation: { source: "controller_lease", expected: false, reason: "Legacy controller lease lost (renewal_rejected)" },
executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } });
expect(f.deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce();
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "failed", expect.any(String), expect.objectContaining({ keepIdleOnFailure: true }));
});
it("attributes a lease loss even when a setup step throws its own abort error", async () => {
const f = await fixture();
f.controller.abort(new LegacyControllerLeaseLostError("renewal_deadline"));
await f.completion.failRunSetup(f.setup(new Error("environment acquisition aborted")));
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "controller_lease_lost",
resultJson: { cancellation: { source: "controller_lease", reason: "Legacy controller lease lost (renewal_deadline)" } } });
});
it("does not make a lease loss retryable once provider work has started", async () => {
const f = await fixture();
f.controller.abort(new LegacyControllerLeaseLostError("renewal_error"));
await f.completion.failRunSetup(f.setup(new Error("usage capture write failed"), true));
expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "setup_failed",
resultJson: { cancellation: { source: "controller_lease", reason: "Legacy controller lease lost (renewal_error)" } } });
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "failed", expect.any(String), expect.objectContaining({ keepIdleOnFailure: false }));
});
it("leaves a winning terminal outcome and agent state untouched after a late setup error", async () => {
const f = await fixture({ status: "succeeded", resultJson: { winner: "completion" } });
await f.completion.failRunSetup(f.setup(new Error("late setup error")));
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", resultJson: { winner: "completion" } });
expect(f.deps.setWakeupStatus).not.toHaveBeenCalled();
expect(f.deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled();
expect(f.deps.finalizeAgentStatus).not.toHaveBeenCalled();
});
});