The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
import { and, asc, eq, inArray, isNull, lt, lte, or, sql } from "drizzle-orm";
import { executionWorkspaceRestorePoints, type Db } from "@paperclipai/db";
import type { StorageService } from "../storage/types.js";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export const GIT_RESTORE_POINT_RETRY_MS = 15 * 60 * 1000;
export const GIT_RESTORE_POINT_DEFAULT_RETENTION_DAYS = 90;
const DAY_MS = 24 * 60 * 60 * 1000;
const DEFAULT_PENDING_GRACE_MS = 60 * 60 * 1000;
const DEFAULT_BATCH = 50;
type RestorePoint = typeof executionWorkspaceRestorePoints.$inferSelect;
export type GitRestorePointRetentionResult = {
abandoned: number;
scheduled: number;
expired: number;
orphaned: number;
deleted: number;
failed: number;
raced: number;
truncated: boolean;
};
export type GitRestorePointRetentionOptions = {
storage: StorageService;
retentionDays?: number;
pendingGraceMs?: number;
batchSize?: number;
now?: () => Date;
beforeExpire?: (restorePoint: RestorePoint) => Promise<void>;
afterTreeRead?: (restorePoint: RestorePoint) => Promise<void>;
};
type TreeState = { exists: boolean; open: boolean; closedAt: Date | null };
export function gitRestorePointRetentionService(db: Db, options: GitRestorePointRetentionOptions) {
const retentionMs = Math.max(1, options.retentionDays ?? GIT_RESTORE_POINT_DEFAULT_RETENTION_DAYS) * DAY_MS;
const pendingGraceMs = options.pendingGraceMs ?? DEFAULT_PENDING_GRACE_MS;
const batchSize = options.batchSize ?? DEFAULT_BATCH;
const now = options.now ?? (() => new Date());
async function issueTreeState(row: RestorePoint): Promise<TreeState> {
if (!row.sourceIssueId) return { exists: false, open: false, closedAt: null };
const result = await db.execute<{ total: number; open: number; closed_at: string | null }>(sql`
WITH RECURSIVE tree AS (
SELECT id, status, completed_at, cancelled_at, updated_at FROM issues
WHERE company_id = ${row.companyId} AND id = ${row.sourceIssueId}
UNION
SELECT c.id, c.status, c.completed_at, c.cancelled_at, c.updated_at FROM issues c
JOIN tree p ON c.parent_id = p.id WHERE c.company_id = ${row.companyId}
)
SELECT count(*)::int AS total,
count(*) FILTER (WHERE status NOT IN ('done', 'cancelled'))::int AS open,
max(CASE WHEN status = 'done' THEN coalesce(completed_at, updated_at)
WHEN status = 'cancelled' THEN coalesce(cancelled_at, updated_at) END) AS closed_at
FROM tree`);
const state = result[0];
if (!state || state.total === 0) return { exists: false, open: false, closedAt: null };
return { exists: true, open: state.open > 0, closedAt: state.closed_at ? new Date(state.closed_at) : null };
}
async function requestDeletion(ids: string[], at: Date) {
if (ids.length === 0) return;
await db.update(executionWorkspaceRestorePoints)
.set({ status: "delete_requested", updatedAt: at })
.where(and(inArray(executionWorkspaceRestorePoints.id, ids),
inArray(executionWorkspaceRestorePoints.status, ["pending", "active"])));
}
async function evaluateActive(row: RestorePoint, at: Date, result: GitRestorePointRetentionResult) {
const tree = await issueTreeState(row);
await options.afterTreeRead?.(row);
const unchanged = and(
eq(executionWorkspaceRestorePoints.id, row.id),
eq(executionWorkspaceRestorePoints.status, "active"),
eq(executionWorkspaceRestorePoints.reviewGeneration, row.reviewGeneration),
);
const retainedFrom = tree.closedAt && row.activatedAt && row.objectKey.includes("/task-directory/")
? Math.max(tree.closedAt.getTime(), row.activatedAt.getTime()) : tree.closedAt?.getTime();
const deleteAfter = !tree.exists || tree.open || retainedFrom === undefined ? null : new Date(retainedFrom + retentionMs);
if (!tree.exists || (deleteAfter && deleteAfter.getTime() <= at.getTime())) {
const claimed = await db.update(executionWorkspaceRestorePoints)
.set({ status: tree.exists ? "expiring" : "delete_requested", lastError: null, updatedAt: at })
.where(unchanged)
.returning();
if (claimed.length === 0) {
result.raced += 1;
return;
}
if (!tree.exists) {
result.orphaned += 1;
return;
}
await finishExpiry(claimed[0]!, at, result);
return;
}
const updated = await db.update(executionWorkspaceRestorePoints)
.set({ treeClosedAt: tree.open ? null : tree.closedAt, deleteAfter, updatedAt: at })
.where(unchanged)
.returning({ id: executionWorkspaceRestorePoints.id });
if (updated.length === 0) result.raced += 1;
else result.scheduled += 1;
}
async function finishExpiry(row: RestorePoint, at: Date, result: GitRestorePointRetentionResult) {
try {
await options.beforeExpire?.(row);
} catch (error) {
await db.update(executionWorkspaceRestorePoints)
.set({ lastError: error instanceof Error ? error.message.slice(0, 500) : "expire_failed", updatedAt: at })
.where(and(eq(executionWorkspaceRestorePoints.id, row.id), eq(executionWorkspaceRestorePoints.status, "expiring")));
result.failed += 1;
return;
}
await db.update(executionWorkspaceRestorePoints)
.set({ status: "delete_requested", lastError: null, updatedAt: at })
.where(and(eq(executionWorkspaceRestorePoints.id, row.id), eq(executionWorkspaceRestorePoints.status, "expiring")));
result.expired += 1;
}
function retryable(status: "expiring" | "superseded" | "delete_requested" | Array<"superseded" | "delete_requested">, at: Date) {
return and(
Array.isArray(status) ? inArray(executionWorkspaceRestorePoints.status, status) : eq(executionWorkspaceRestorePoints.status, status),
or(isNull(executionWorkspaceRestorePoints.lastError),
lte(executionWorkspaceRestorePoints.updatedAt, new Date(at.getTime() - GIT_RESTORE_POINT_RETRY_MS))),
);
}
async function tick() {
const at = now();
const result: GitRestorePointRetentionResult = { abandoned: 0, scheduled: 0, expired: 0, orphaned: 0, deleted: 0, failed: 0, raced: 0, truncated: false };
const abandoned = await db.select({ id: executionWorkspaceRestorePoints.id }).from(executionWorkspaceRestorePoints)
.where(and(eq(executionWorkspaceRestorePoints.status, "pending"),
lt(executionWorkspaceRestorePoints.createdAt, new Date(at.getTime() - pendingGraceMs))))
.orderBy(asc(executionWorkspaceRestorePoints.createdAt)).limit(batchSize);
await requestDeletion(abandoned.map((row) => row.id), at);
result.abandoned = abandoned.length;
const due = await db.select().from(executionWorkspaceRestorePoints)
.where(and(eq(executionWorkspaceRestorePoints.status, "active"),
sql`${executionWorkspaceRestorePoints.deleteAfter} <= greatest(${at.toISOString()}::timestamptz, clock_timestamp())`))
.orderBy(asc(executionWorkspaceRestorePoints.deleteAfter), asc(executionWorkspaceRestorePoints.id)).limit(batchSize);
for (const row of due) await evaluateActive(row, at, result);
const expiring = await db.select().from(executionWorkspaceRestorePoints)
.where(retryable("expiring", at))
.orderBy(asc(executionWorkspaceRestorePoints.updatedAt), asc(executionWorkspaceRestorePoints.id)).limit(batchSize);
for (const row of expiring) await finishExpiry(row, at, result);
const doomed = await db.select().from(executionWorkspaceRestorePoints)
.where(retryable(["superseded", "delete_requested"], at))
.orderBy(asc(executionWorkspaceRestorePoints.updatedAt), asc(executionWorkspaceRestorePoints.id)).limit(batchSize);
for (const row of doomed) {
try {
await options.storage.deleteObject(row.companyId, row.objectKey);
if ((await options.storage.headObject(row.companyId, row.objectKey)).exists) throw new Error("restore_point_object_still_present");
await db.update(executionWorkspaceRestorePoints)
.set({ status: "deleted", deletedAt: at, lastError: null, updatedAt: at })
.where(eq(executionWorkspaceRestorePoints.id, row.id));
result.deleted += 1;
} catch (error) {
await db.update(executionWorkspaceRestorePoints)
.set({ lastError: error instanceof Error ? error.message.slice(0, 500) : "delete_failed", updatedAt: at })
.where(eq(executionWorkspaceRestorePoints.id, row.id));
result.failed += 1;
}
}
result.truncated = abandoned.length === batchSize || due.length === batchSize
|| expiring.length === batchSize || doomed.length === batchSize;
return result;
}
async function nextDeadlineAt(): Promise<number | null> {
const [row] = await db.select({
deleteNow: sql<number>`count(*) FILTER (WHERE ${executionWorkspaceRestorePoints.status} IN ('expiring', 'superseded', 'delete_requested') AND ${executionWorkspaceRestorePoints.lastError} IS NULL)::int`,
retryFrom: sql<string | null>`min(${executionWorkspaceRestorePoints.updatedAt}) FILTER (WHERE ${executionWorkspaceRestorePoints.status} IN ('expiring', 'superseded', 'delete_requested') AND ${executionWorkspaceRestorePoints.lastError} IS NOT NULL)`,
pendingFrom: sql<string | null>`min(${executionWorkspaceRestorePoints.createdAt}) FILTER (WHERE ${executionWorkspaceRestorePoints.status} = 'pending')`,
expiresAt: sql<string | null>`min(${executionWorkspaceRestorePoints.deleteAfter}) FILTER (WHERE ${executionWorkspaceRestorePoints.status} = 'active')`,
}).from(executionWorkspaceRestorePoints);
const at = now().getTime();
const deadlines: number[] = [];
if (row && row.deleteNow > 0) deadlines.push(at);
if (row?.retryFrom) deadlines.push(new Date(row.retryFrom).getTime() + GIT_RESTORE_POINT_RETRY_MS);
if (row?.pendingFrom) deadlines.push(new Date(row.pendingFrom).getTime() + pendingGraceMs);
if (row?.expiresAt) deadlines.push(new Date(row.expiresAt).getTime());
return deadlines.length > 0 ? Math.min(...deadlines) : null;
}
async function hasActive(): Promise<boolean> {
const [row] = await db.select({ id: executionWorkspaceRestorePoints.id }).from(executionWorkspaceRestorePoints)
.where(eq(executionWorkspaceRestorePoints.status, "active")).limit(1);
return Boolean(row);
}
return { tick, nextDeadlineAt, hasActive };
}
export function notifyGitRestorePointWork(transaction: object, hint?: string) {
return notifyDeliveryWork(transaction, DELIVERY_QUEUES.gitRestorePoints, hint);
}
export async function requestRestorePointDeletion(
tx: Pick<Db, "update">,
scope: { sourceIssueId: string } | { companyId: string },
) {
await notifyGitRestorePointWork(tx, "sourceIssueId" in scope ? `issue:${scope.sourceIssueId}` : `company:${scope.companyId}`);
await tx.update(executionWorkspaceRestorePoints)
.set({ status: "delete_requested", updatedAt: new Date() })
.where(and(
"sourceIssueId" in scope
? eq(executionWorkspaceRestorePoints.sourceIssueId, scope.sourceIssueId)
: eq(executionWorkspaceRestorePoints.companyId, scope.companyId),
inArray(executionWorkspaceRestorePoints.status, ["pending", "active", "expiring"]),
));
}
export function registerGitRestorePointWork(
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
service: ReturnType<typeof gitRestorePointRetentionService>,
onResult: (result: GitRestorePointRetentionResult) => void,
) {
let next: number | null = null;
let tracksActive = true;
const worker = coordinator.register(DELIVERY_QUEUES.gitRestorePoints, {
retryMs: GIT_RESTORE_POINT_RETRY_MS,
run: async () => {
next = null;
const result = await service.tick();
onResult(result);
next = result.truncated ? Date.now() + 1000 : await service.nextDeadlineAt();
tracksActive = await service.hasActive();
},
hasPending: async () => false,
nextRunAt: () => next,
});
return { ...worker, issuesChanged: () => { if (tracksActive) worker.wake(); } };
}