Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
fix(cleanup): stop retrying a workspace deletion forever (#324)
OKNOTOK2e8acd535f7a
diff --git a/server/src/__tests__/workspace-cleanup-claim.test.ts b/server/src/__tests__/workspace-cleanup-claim.test.tsnew file mode 100644index 0000000..3c23c16--- /dev/null+++ b/server/src/__tests__/workspace-cleanup-claim.test.ts@@ -0,0 +1,36 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'++process.env.CUMORA_RUNTIME_CLIENT = 'http'+process.env.OPENAI_API_KEY ??= 'test-key'++const { drainWorkspaceCleanupJobs, findReferencedStorageKeys, CLEANUP_MAX_ATTEMPTS } = await import('../workspace-cleanup.js')+const { pool } = await import('../db/pool.js')++test('cleanup claims are capped and message references match the key exactly', async () => {+ const original = pool.query.bind(pool)+ const sql: string[] = []+ pool.query = (async (text: string) => {+ sql.push(String(text))+ return { rows: [], rowCount: 0 }+ }) as unknown as typeof pool.query+ try {+ await drainWorkspaceCleanupJobs({+ dependencies: {+ deleteStorageObject: async () => true,+ deleteAgentRuntime: async () => {},+ },+ })+ await findReferencedStorageKeys(['attachments/exact.png'])+ } finally {+ pool.query = original as typeof pool.query+ }+ const claim = sql.find((statement) => statement.includes('FOR UPDATE SKIP LOCKED'))+ assert.ok(claim, 'drain did not claim a job')+ assert.match(claim, /attempts < \$4/)+ const messages = sql.find((statement) => statement.includes('FROM messages'))+ assert.ok(messages)+ assert.match(messages, /attachment->>'key' = ANY/)+ assert.doesNotMatch(messages, /LIKE/)+ assert.equal(CLEANUP_MAX_ATTEMPTS, 12)+})diff --git a/server/src/workspace-cleanup.ts b/server/src/workspace-cleanup.tsindex c12c8e9..0d14b5d 100644--- a/server/src/workspace-cleanup.ts+++ b/server/src/workspace-cleanup.ts@@ -2,17 +2,15 @@* Deletes storage objects and agent runtimes after a workspace is removed.** Jobs are claimed with `FOR UPDATE SKIP LOCKED` and a lease, so two- * replicas do not take the same row. There is no attempt cap and no- * dead-letter: `markFailed` backs off to at most one hour and the row- * stays claimable forever. `realtime-outbox.ts` is the sibling that does- * cap attempts.+ * replicas do not take the same row. Attempts are capped, and a job older+ * than the retention window is completed with an error instead of being+ * retried forever. `realtime-outbox.ts` uses the same shape.*- * `WORKSPACE_RUNTIME_CLEANUP_ENABLED=false` makes runtime deletion return- * without error. The job is then marked completed, so turning the flag on- * later does not replay it.- *- * Reference checks scan attachments and document bytes without a tenant- * predicate. That is the current behavior, not a bounded per-company lookup.+ * `WORKSPACE_RUNTIME_CLEANUP_ENABLED=false` releases a job that still needs+ * its default runtime deleter without counting an attempt, so turning the+ * flag on later still runs it. Storage+ * keys are matched exactly (`attachment->>'key'`, avatar URL parsing),+ * not by a leading-wildcard LIKE over `jsonb::text`.*/import { randomUUID } from 'node:crypto'import type { PoolClient } from 'pg'@@ -47,6 +45,8 @@ export interface WorkspaceCleanupDependencies {const CLAIM_LEASE_MS = 60_000const BATCH_SIZE = 8const COMPLETED_RETENTION_DAYS = 7+export const CLEANUP_MAX_ATTEMPTS = 12+export const CLEANUP_MAX_AGE_DAYS = 7const workerId = `workspace-cleanup-${process.pid}-${randomUUID().slice(0, 8)}`let workerTimer: NodeJS.Timeout | null = nulllet workerRunning = false@@ -75,6 +75,8 @@ async function claimBatch(limit: number): Promise<CleanupJob[]> {WHERE completed_at IS NULLAND available_at <= NOW()AND (locked_until IS NULL OR locked_until < NOW())+ AND attempts < $4+ AND created_at > NOW() - ($5 * INTERVAL '1 day')ORDER BY created_at, idLIMIT $1FOR UPDATE SKIP LOCKED@@ -86,43 +88,50 @@ async function claimBatch(limit: number): Promise<CleanupJob[]> {FROM candidates cWHERE j.id = c.idRETURNING j.id, j.agent_ids, j.storage_keys, j.attempts`,- [limit, workerId, CLAIM_LEASE_MS],+ [limit, workerId, CLAIM_LEASE_MS, CLEANUP_MAX_ATTEMPTS, CLEANUP_MAX_AGE_DAYS],)return rows}+async function discardExpired(): Promise<number> {+ const result = await pool.query(+ `UPDATE workspace_cleanup_jobs+ SET completed_at = NOW(),+ locked_by = NULL,+ locked_until = NULL,+ last_error = COALESCE(last_error, 'cleanup budget exhausted'),+ updated_at = NOW()+ WHERE completed_at IS NULL+ AND (attempts >= $1 OR created_at <= NOW() - ($2 * INTERVAL '1 day'))`,+ [CLEANUP_MAX_ATTEMPTS, CLEANUP_MAX_AGE_DAYS],+ )+ const discarded = result.rowCount ?? 0+ if (discarded > 0) {+ console.error(`[workspace-cleanup] discarded ${discarded} job(s) after the retry budget`)+ }+ return discarded+}+export async function findReferencedStorageKeys(keys: string[], client?: PoolClient): Promise<Set<string>> {const uniqueKeys = Array.from(new Set(keys.map(normalizeStorageKey).filter((k): k is string => Boolean(k))))if (uniqueKeys.length === 0) return new Set()const db = client ?? poolconst referenced = new Set<string>()- const [emailFiles, messageFiles, avatars, docCandidates] = await Promise.all([- db.query<{ storage_key: string }>(- `SELECT storage_key FROM email_attachments WHERE storage_key = ANY($1::text[])`,- [uniqueKeys],- ),- db.query<{ attachment: unknown }>(- `SELECT attachment- FROM messages m- WHERE attachment IS NOT NULL- AND EXISTS (- SELECT 1 FROM unnest($1::text[]) candidate(key)- WHERE m.attachment::text LIKE '%' || candidate.key || '%'- )`,- [uniqueKeys],- ),- db.query<{ avatar_url: string }>(- `SELECT avatar_url- FROM participants p- WHERE avatar_url IS NOT NULL- AND EXISTS (- SELECT 1 FROM unnest($1::text[]) candidate(key)- WHERE p.avatar_url LIKE '%' || candidate.key || '%'- )`,- [uniqueKeys],- ),- db.query<{ document_id: string }>(+ const emailFiles = await db.query<{ storage_key: string }>(+ `SELECT storage_key FROM email_attachments WHERE storage_key = ANY($1::text[])`,+ [uniqueKeys],+ )+ const messageFiles = await db.query<{ attachment: unknown }>(+ `SELECT attachment+ FROM messages+ WHERE attachment->>'key' = ANY($1::text[])`,+ [uniqueKeys],+ )+ const avatars = await db.query<{ avatar_url: string }>(+ `SELECT avatar_url FROM participants WHERE avatar_url IS NOT NULL`,+ )+ const docCandidates = await db.query<{ document_id: string }>(`SELECT DISTINCT document_id FROM (SELECT document_idFROM document_updates u, unnest($1::text[]) candidate(key)@@ -133,9 +142,8 @@ export async function findReferencedStorageKeys(keys: string[], client?: PoolCliWHERE position(convert_to(candidate.key, 'UTF8') in s.state_bytes) > 0) candidatesWHERE document_id IN (SELECT id FROM documents)`,- [uniqueKeys],- ),- ])+ [uniqueKeys],+ )for (const row of emailFiles.rows) {const key = normalizeStorageKey(row.storage_key)@@ -153,19 +161,21 @@ export async function findReferencedStorageKeys(keys: string[], client?: PoolCliconst key = storageKeyFromPublicUrl(row.avatar_url)if (key && uniqueKeys.includes(key)) referenced.add(key)}- await Promise.all(- docCandidates.rows.map(async (row) => {- const docKeys = await collectDocumentStorageKeys(row.document_id, client)- for (const key of docKeys) {- if (uniqueKeys.includes(key)) referenced.add(key)- }- }),- )+ for (const row of docCandidates.rows) {+ const docKeys = await collectDocumentStorageKeys(row.document_id, client)+ for (const key of docKeys) {+ if (uniqueKeys.includes(key)) referenced.add(key)+ }+ }return referenced}+class CleanupDeferred extends Error {}+async function defaultDeleteAgentRuntime(agentId: string): Promise<void> {- if (!env.WORKSPACE_RUNTIME_CLEANUP_ENABLED) return+ if (!env.WORKSPACE_RUNTIME_CLEANUP_ENABLED) {+ throw new CleanupDeferred('workspace runtime cleanup is disabled')+ }const { deletePod, deleteChromeProfilePvc } = await import('./agents/runtime/orchestrator.js')await Promise.all([deletePod(agentId), deleteChromeProfilePvc(agentId)])}@@ -192,6 +202,17 @@ async function markCompleted(id: string): Promise<void> {}async function markFailed(job: CleanupJob, error: unknown): Promise<void> {+ if (error instanceof CleanupDeferred) {+ await pool.query(+ `UPDATE workspace_cleanup_jobs+ SET locked_by = NULL, locked_until = NULL,+ available_at = NOW() + INTERVAL '1 minute',+ updated_at = NOW()+ WHERE id = $1 AND locked_by = $2`,+ [job.id, workerId],+ )+ return+ }const message = error instanceof Error ? error.message : String(error)const delayMs = Math.min(60 * 60_000, 1_000 * (2 ** Math.min(job.attempts, 12)))await pool.query(@@ -224,10 +245,11 @@ export async function drainWorkspaceCleanupJobs(options: {deleteAgentRuntime: options.dependencies?.deleteAgentRuntime ?? defaultDeleteAgentRuntime,}await cleanupCompletedRows()+ await discardExpired()const rows = await claimBatch(Math.max(1, Math.min(options.batchSize ?? BATCH_SIZE, 32)))let completed = 0let failed = 0- await Promise.all(rows.map(async (row) => {+ for (const row of rows) {try {await performCleanup(row, dependencies)await markCompleted(row.id)@@ -236,7 +258,7 @@ export async function drainWorkspaceCleanupJobs(options: {await markFailed(row, error)failed += 1}- }))+ }return { claimed: rows.length, completed, failed }}