Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(cleanup): stop retrying a workspace deletion forever (#324)

OKNOTOK2e8acd535f7a
diff --git a/server/src/__tests__/workspace-cleanup-claim.test.ts b/server/src/__tests__/workspace-cleanup-claim.test.ts
new file mode 100644
index 0000000..3c23c16
--- /dev/null
+++ b/server/src/__tests__/workspace-cleanup-claim.test.ts
@@ -0,0 +1,36 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+process.env.CUMORA_RUNTIME_CLIENT = 'http'
+process.env.OPENAI_API_KEY ??= 'test-key'
+
+const { drainWorkspaceCleanupJobs, findReferencedStorageKeys, CLEANUP_MAX_ATTEMPTS } = await import('../workspace-cleanup.js')
+const { pool } = await import('../db/pool.js')
+
+test('cleanup claims are capped and message references match the key exactly', async () => {
+ const original = pool.query.bind(pool)
+ const sql: string[] = []
+ pool.query = (async (text: string) => {
+ sql.push(String(text))
+ return { rows: [], rowCount: 0 }
+ }) as unknown as typeof pool.query
+ try {
+ await drainWorkspaceCleanupJobs({
+ dependencies: {
+ deleteStorageObject: async () => true,
+ deleteAgentRuntime: async () => {},
+ },
+ })
+ await findReferencedStorageKeys(['attachments/exact.png'])
+ } finally {
+ pool.query = original as typeof pool.query
+ }
+ const claim = sql.find((statement) => statement.includes('FOR UPDATE SKIP LOCKED'))
+ assert.ok(claim, 'drain did not claim a job')
+ assert.match(claim, /attempts < \$4/)
+ const messages = sql.find((statement) => statement.includes('FROM messages'))
+ assert.ok(messages)
+ assert.match(messages, /attachment->>'key' = ANY/)
+ assert.doesNotMatch(messages, /LIKE/)
+ assert.equal(CLEANUP_MAX_ATTEMPTS, 12)
+})
diff --git a/server/src/workspace-cleanup.ts b/server/src/workspace-cleanup.ts
index c12c8e9..0d14b5d 100644
--- a/server/src/workspace-cleanup.ts
+++ b/server/src/workspace-cleanup.ts
@@ -2,17 +2,15 @@
* Deletes storage objects and agent runtimes after a workspace is removed.
*
* Jobs are claimed with `FOR UPDATE SKIP LOCKED` and a lease, so two
- * replicas do not take the same row. There is no attempt cap and no
- * dead-letter: `markFailed` backs off to at most one hour and the row
- * stays claimable forever. `realtime-outbox.ts` is the sibling that does
- * cap attempts.
+ * replicas do not take the same row. Attempts are capped, and a job older
+ * than the retention window is completed with an error instead of being
+ * retried forever. `realtime-outbox.ts` uses the same shape.
*
- * `WORKSPACE_RUNTIME_CLEANUP_ENABLED=false` makes runtime deletion return
- * without error. The job is then marked completed, so turning the flag on
- * later does not replay it.
- *
- * Reference checks scan attachments and document bytes without a tenant
- * predicate. That is the current behavior, not a bounded per-company lookup.
+ * `WORKSPACE_RUNTIME_CLEANUP_ENABLED=false` releases a job that still needs
+ * its default runtime deleter without counting an attempt, so turning the
+ * flag on later still runs it. Storage
+ * keys are matched exactly (`attachment->>'key'`, avatar URL parsing),
+ * not by a leading-wildcard LIKE over `jsonb::text`.
*/
import { randomUUID } from 'node:crypto'
import type { PoolClient } from 'pg'
@@ -47,6 +45,8 @@ export interface WorkspaceCleanupDependencies {
const CLAIM_LEASE_MS = 60_000
const BATCH_SIZE = 8
const COMPLETED_RETENTION_DAYS = 7
+export const CLEANUP_MAX_ATTEMPTS = 12
+export const CLEANUP_MAX_AGE_DAYS = 7
const workerId = `workspace-cleanup-${process.pid}-${randomUUID().slice(0, 8)}`
let workerTimer: NodeJS.Timeout | null = null
let workerRunning = false
@@ -75,6 +75,8 @@ async function claimBatch(limit: number): Promise<CleanupJob[]> {
WHERE completed_at IS NULL
AND available_at <= NOW()
AND (locked_until IS NULL OR locked_until < NOW())
+ AND attempts < $4
+ AND created_at > NOW() - ($5 * INTERVAL '1 day')
ORDER BY created_at, id
LIMIT $1
FOR UPDATE SKIP LOCKED
@@ -86,43 +88,50 @@ async function claimBatch(limit: number): Promise<CleanupJob[]> {
FROM candidates c
WHERE j.id = c.id
RETURNING j.id, j.agent_ids, j.storage_keys, j.attempts`,
- [limit, workerId, CLAIM_LEASE_MS],
+ [limit, workerId, CLAIM_LEASE_MS, CLEANUP_MAX_ATTEMPTS, CLEANUP_MAX_AGE_DAYS],
)
return rows
}
+async function discardExpired(): Promise<number> {
+ const result = await pool.query(
+ `UPDATE workspace_cleanup_jobs
+ SET completed_at = NOW(),
+ locked_by = NULL,
+ locked_until = NULL,
+ last_error = COALESCE(last_error, 'cleanup budget exhausted'),
+ updated_at = NOW()
+ WHERE completed_at IS NULL
+ AND (attempts >= $1 OR created_at <= NOW() - ($2 * INTERVAL '1 day'))`,
+ [CLEANUP_MAX_ATTEMPTS, CLEANUP_MAX_AGE_DAYS],
+ )
+ const discarded = result.rowCount ?? 0
+ if (discarded > 0) {
+ console.error(`[workspace-cleanup] discarded ${discarded} job(s) after the retry budget`)
+ }
+ return discarded
+}
+
export async function findReferencedStorageKeys(keys: string[], client?: PoolClient): Promise<Set<string>> {
const uniqueKeys = Array.from(new Set(keys.map(normalizeStorageKey).filter((k): k is string => Boolean(k))))
if (uniqueKeys.length === 0) return new Set()
const db = client ?? pool
const referenced = new Set<string>()
- const [emailFiles, messageFiles, avatars, docCandidates] = await Promise.all([
- db.query<{ storage_key: string }>(
- `SELECT storage_key FROM email_attachments WHERE storage_key = ANY($1::text[])`,
- [uniqueKeys],
- ),
- db.query<{ attachment: unknown }>(
- `SELECT attachment
- FROM messages m
- WHERE attachment IS NOT NULL
- AND EXISTS (
- SELECT 1 FROM unnest($1::text[]) candidate(key)
- WHERE m.attachment::text LIKE '%' || candidate.key || '%'
- )`,
- [uniqueKeys],
- ),
- db.query<{ avatar_url: string }>(
- `SELECT avatar_url
- FROM participants p
- WHERE avatar_url IS NOT NULL
- AND EXISTS (
- SELECT 1 FROM unnest($1::text[]) candidate(key)
- WHERE p.avatar_url LIKE '%' || candidate.key || '%'
- )`,
- [uniqueKeys],
- ),
- db.query<{ document_id: string }>(
+ const emailFiles = await db.query<{ storage_key: string }>(
+ `SELECT storage_key FROM email_attachments WHERE storage_key = ANY($1::text[])`,
+ [uniqueKeys],
+ )
+ const messageFiles = await db.query<{ attachment: unknown }>(
+ `SELECT attachment
+ FROM messages
+ WHERE attachment->>'key' = ANY($1::text[])`,
+ [uniqueKeys],
+ )
+ const avatars = await db.query<{ avatar_url: string }>(
+ `SELECT avatar_url FROM participants WHERE avatar_url IS NOT NULL`,
+ )
+ const docCandidates = await db.query<{ document_id: string }>(
`SELECT DISTINCT document_id FROM (
SELECT document_id
FROM document_updates u, unnest($1::text[]) candidate(key)
@@ -133,9 +142,8 @@ export async function findReferencedStorageKeys(keys: string[], client?: PoolCli
WHERE position(convert_to(candidate.key, 'UTF8') in s.state_bytes) > 0
) candidates
WHERE document_id IN (SELECT id FROM documents)`,
- [uniqueKeys],
- ),
- ])
+ [uniqueKeys],
+ )
for (const row of emailFiles.rows) {
const key = normalizeStorageKey(row.storage_key)
@@ -153,19 +161,21 @@ export async function findReferencedStorageKeys(keys: string[], client?: PoolCli
const key = storageKeyFromPublicUrl(row.avatar_url)
if (key && uniqueKeys.includes(key)) referenced.add(key)
}
- await Promise.all(
- docCandidates.rows.map(async (row) => {
- const docKeys = await collectDocumentStorageKeys(row.document_id, client)
- for (const key of docKeys) {
- if (uniqueKeys.includes(key)) referenced.add(key)
- }
- }),
- )
+ for (const row of docCandidates.rows) {
+ const docKeys = await collectDocumentStorageKeys(row.document_id, client)
+ for (const key of docKeys) {
+ if (uniqueKeys.includes(key)) referenced.add(key)
+ }
+ }
return referenced
}
+class CleanupDeferred extends Error {}
+
async function defaultDeleteAgentRuntime(agentId: string): Promise<void> {
- if (!env.WORKSPACE_RUNTIME_CLEANUP_ENABLED) return
+ if (!env.WORKSPACE_RUNTIME_CLEANUP_ENABLED) {
+ throw new CleanupDeferred('workspace runtime cleanup is disabled')
+ }
const { deletePod, deleteChromeProfilePvc } = await import('./agents/runtime/orchestrator.js')
await Promise.all([deletePod(agentId), deleteChromeProfilePvc(agentId)])
}
@@ -192,6 +202,17 @@ async function markCompleted(id: string): Promise<void> {
}
async function markFailed(job: CleanupJob, error: unknown): Promise<void> {
+ if (error instanceof CleanupDeferred) {
+ await pool.query(
+ `UPDATE workspace_cleanup_jobs
+ SET locked_by = NULL, locked_until = NULL,
+ available_at = NOW() + INTERVAL '1 minute',
+ updated_at = NOW()
+ WHERE id = $1 AND locked_by = $2`,
+ [job.id, workerId],
+ )
+ return
+ }
const message = error instanceof Error ? error.message : String(error)
const delayMs = Math.min(60 * 60_000, 1_000 * (2 ** Math.min(job.attempts, 12)))
await pool.query(
@@ -224,10 +245,11 @@ export async function drainWorkspaceCleanupJobs(options: {
deleteAgentRuntime: options.dependencies?.deleteAgentRuntime ?? defaultDeleteAgentRuntime,
}
await cleanupCompletedRows()
+ await discardExpired()
const rows = await claimBatch(Math.max(1, Math.min(options.batchSize ?? BATCH_SIZE, 32)))
let completed = 0
let failed = 0
- await Promise.all(rows.map(async (row) => {
+ for (const row of rows) {
try {
await performCleanup(row, dependencies)
await markCompleted(row.id)
@@ -236,7 +258,7 @@ export async function drainWorkspaceCleanupJobs(options: {
await markFailed(row, error)
failed += 1
}
- }))
+ }
return { claimed: rows.length, completed, failed }
}