The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
import { readProcessStartedAt } from "./hot-restart.js";
import { runUsedConversationAdapter } from "./conversation-continuation.js";
import { and, eq, inArray, isNull, or, sql } from "drizzle-orm";
import { environmentLeases, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { hasNativeLocalProcessStop } from "./native-local-process-stop.js";
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
import { isCancelledNativeStartup } from "./cancelled-native-startup.js";
/** Cancellation changes admission. Process and lease evidence controls cleanup. */
export async function agentExecutionsHaveStopped(db: Db, agentIds: string[]) {
if (!agentIds.length) return true;
const runs = await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds));
const runIds = db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(inArray(heartbeatRuns.agentId, agentIds));
const leases = await db.select().from(environmentLeases).where(or(
inArray(environmentLeases.heartbeatRunId, runIds),
inArray(sql`${environmentLeases.metadata}->>'agentId'`, agentIds),
));
if (leases.some(lease => (!lease.releasedAt && !(lease.status === "retained" && lease.cleanupStatus === "success")) || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed")) return false;
const coordinators = await db.select().from(nativeRunFinalizations)
.where(inArray(nativeRunFinalizations.runId, runIds));
const now = new Date();
const abandoned = coordinators.filter(row => row.leaseOwner);
if (abandoned.some(row => !row.leaseExpiresAt || row.leaseExpiresAt > now || row.resultId ||
runs.find(run => run.id === row.runId)?.status !== "cancelled")) return false;
const absent = (pid: number) => {
try { process.kill(pid, 0); return false; }
catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH"; }
};
for (const coordinator of abandoned) {
if (!coordinator.controllerPid || absent(coordinator.controllerPid)) continue;
const observed = await readProcessStartedAt(coordinator.controllerPid).catch(() => null);
if (!observed || !coordinator.controllerProcessStartedAt ||
new Date(observed).getTime() === coordinator.controllerProcessStartedAt.getTime()) return false;
}
for (const run of runs) {
if (!["succeeded", "failed", "cancelled", "timed_out", "interrupted"].includes(run.status)) return false;
if (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()) return false;
const runLeases = leases.filter(lease => lease.heartbeatRunId === run.id);
if (runLeases.some(lease => lease.provider && lease.provider !== "local")) {
if (run.status === "cancelled" && !runLeases.every(hasRemoteTerminationReceipt)) return false;
continue;
}
let processAlive = Boolean(run.processPid && !absent(run.processPid));
let groupAlive = Boolean(run.processGroupId && !absent(-run.processGroupId));
if (processAlive && run.processStartedAt) {
const observed = await readProcessStartedAt(run.processPid!).catch(() => null);
if (observed && new Date(observed).getTime() !== run.processStartedAt.getTime()) {
processAlive = false;
if (run.processGroupId === run.processPid) groupAlive = false;
}
}
if (processAlive || groupAlive) return false;
if (run.status !== "cancelled" || !run.startedAt) continue;
if (run.runtimeMode === "native" && !run.processPid && !run.processGroupId &&
!await hasNativeLocalProcessStop(db, run.companyId, run.id) &&
!await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return false;
if (run.runtimeMode === "legacy" && !run.processPid && !run.processGroupId &&
(run.resultJson?.executionCancellation as Record<string, unknown> | undefined)?.state !== "acknowledged" &&
!(run.executionStage === "settled" && !await runUsedConversationAdapter(db, run)) &&
!await isCancelledNativeStartup(db, run, coordinators.find(row => row.runId === run.id))) return false;
}
// Release an expired owner only after its cancelled execution has stop proof.
for (const coordinator of abandoned) {
const [cleared] = await db.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: null, updatedAt: now })
.where(and(eq(nativeRunFinalizations.runId, coordinator.runId),
eq(nativeRunFinalizations.leaseOwner, coordinator.leaseOwner!),
eq(nativeRunFinalizations.leaseExpiresAt, coordinator.leaseExpiresAt!),
eq(nativeRunFinalizations.controllerGeneration, coordinator.controllerGeneration),
eq(nativeRunFinalizations.phase, coordinator.phase), isNull(nativeRunFinalizations.resultId)))
.returning({ runId: nativeRunFinalizations.runId });
if (!cleared) return false;
}
return true;
}