Skip to content

The open-source app everyone uses to manage agents at work

fix(heartbeat): let a live legacy controller finish before the finalization deadline fails its run (#16622)

Devin Foley32ee2e9a6a35
diff --git a/server/src/services/execution-control-reconciliation.test.ts b/server/src/services/execution-control-reconciliation.test.ts
index 292b1795bc..9d46777715 100644
--- a/server/src/services/execution-control-reconciliation.test.ts
+++ b/server/src/services/execution-control-reconciliation.test.ts
@@ -97,6 +97,38 @@ describeEmbeddedPostgres("reconcileAbandonedExecutionControl reports a genuine f
});
});
+ it("leaves a run owned by a live legacy controller to that controller", async () => {
+ const { runId } = await seedAbandonedRunFixture();
+ const leaseExpiresAt = new Date(Date.now() + 45_000);
+ await db
+ .update(heartbeatRuns)
+ .set({ runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: leaseExpiresAt })
+ .where(eq(heartbeatRuns.id, runId));
+ const captureCallsBefore = mockCaptureRunFailure.mock.calls.length;
+
+ const result = await reconcileAbandonedExecutionControl(db);
+
+ expect(result.surfaced).toBe(0);
+ const [owned] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
+ expect(owned?.status).toBe("running");
+ expect(owned?.errorCode).toBeNull();
+ expect(owned?.executionControlDeadlineAt?.getTime()).toBe(leaseExpiresAt.getTime());
+
+ await db
+ .update(heartbeatRuns)
+ .set({ controllerLeaseExpiresAt: new Date(Date.now() - 1_000), executionControlDeadlineAt: new Date(Date.now() - 1_000) })
+ .where(eq(heartbeatRuns.id, runId));
+ const lapsed = await reconcileAbandonedExecutionControl(db);
+
+ expect(lapsed.surfaced).toBe(1);
+ const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
+ expect(failed?.status).toBe("failed");
+ expect(failed?.errorCode).toBe("execution_finalization_deadline_exceeded");
+ await vi.waitFor(() => {
+ expect(mockCaptureRunFailure.mock.calls.slice(captureCallsBefore)).toHaveLength(1);
+ }, { timeout: 5_000 });
+ });
+
it("reports zero events for a repeated sweep over the same already-failed run", async () => {
const { runId } = await seedAbandonedRunFixture();
await reconcileAbandonedExecutionControl(db);
diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts
index b2a0d488dc..72266731df 100644
--- a/server/src/services/execution-control-reconciliation.ts
+++ b/server/src/services/execution-control-reconciliation.ts
@@ -10,6 +10,7 @@ import {
} from "@paperclipai/db";
import { parseIssueExecutionState } from "./issue-execution-policy.js";
import { issueRecoveryActionService } from "./issue-recovery-actions.js";
+import { hasLiveLegacyController } from "./legacy-controller-lease.js";
import { reportRunFailure } from "./run-failure-report.js";
import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js";
@@ -151,6 +152,13 @@ export async function reconcileAbandonedExecutionControl(
.where(eq(nativeRunFinalizations.runId, run.id));
return true;
}
+ if (await hasLiveLegacyController(tx as unknown as Db, run)) {
+ await tx
+ .update(heartbeatRuns)
+ .set({ executionControlDeadlineAt: run.controllerLeaseExpiresAt })
+ .where(eq(heartbeatRuns.id, run.id));
+ return false;
+ }
const cause = "execution_finalization_deadline_exceeded";
const nextAction =
"Inspect the failed run and verify its provider has stopped. Reconcile any uncertain external action before explicitly continuing this task.";