Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(documents): purge document image keys on workspace and document deletion (#206) (#222)

cicada97b39bfd96ab
diff --git a/server/src/__integration__/document-image-cleanup.test.ts b/server/src/__integration__/document-image-cleanup.test.ts
new file mode 100644
index 0000000..7d221b0
--- /dev/null
+++ b/server/src/__integration__/document-image-cleanup.test.ts
@@ -0,0 +1,261 @@
+import assert from 'node:assert/strict'
+import { createServer, type Server } from 'node:http'
+import { after, before, beforeEach, test } from 'node:test'
+import * as Y from 'yjs'
+import { pool } from '../db/pool.js'
+import { drainWorkspaceCleanupJobs, findReferencedStorageKeys } from '../workspace-cleanup.js'
+import { markdownToYXml } from '../documents/markdown.js'
+import { runCli } from '../agents/cli.js'
+import { buildApiTestApp, ensureSchemaOnce, resetAllTables, seedCompanyWithAgent, teardownAll } from './_helpers.js'
+
+const OWNER_ID = 'u-cleanup-owner'
+let ownerServer: Server
+let ownerBase = ''
+
+async function listenFor(userId: string): Promise<{ server: Server; base: string }> {
+ const app = await buildApiTestApp(userId)
+ return new Promise((resolve) => {
+ const server = createServer(app).listen(0, () => {
+ const address = server.address()
+ assert.ok(address && typeof address === 'object')
+ resolve({ server, base: `http://127.0.0.1:${address.port}` })
+ })
+ })
+}
+
+before(async () => {
+ await ensureSchemaOnce()
+ const owner = await listenFor(OWNER_ID)
+ ownerServer = owner.server
+ ownerBase = owner.base
+})
+
+beforeEach(async () => {
+ await resetAllTables()
+})
+
+after(async () => {
+ await teardownAll(ownerServer)
+})
+
+async function seedUser(userId: string): Promise<void> {
+ await pool.query(
+ `INSERT INTO users (id, email, display_name, tier)
+ VALUES ($1, $2, $3, 'pro')
+ ON CONFLICT (id) DO NOTHING`,
+ [userId, `${userId}@test.local`, userId],
+ )
+}
+
+async function seedCompany(companyId: string, ownerId: string): Promise<void> {
+ await pool.query(
+ `INSERT INTO companies (id, name, slug, owner_user_id)
+ VALUES ($1, $2, $3, $4)`,
+ [companyId, `Workspace ${companyId}`, companyId, ownerId],
+ )
+}
+
+async function seedMember(
+ companyId: string,
+ userId: string,
+ role: 'owner' | 'admin' | 'member',
+): Promise<void> {
+ await seedUser(userId)
+ await pool.query(
+ `INSERT INTO company_members (company_id, user_id, role) VALUES ($1, $2, $3)`,
+ [companyId, userId, role],
+ )
+ await pool.query(
+ `INSERT INTO participants
+ (id, company_id, kind, name, role, initial, avatar_bg, status)
+ VALUES ($1, $2, 'human', $3, NULL, $4, '#abcdef', 'avail')`,
+ [userId, companyId, userId, userId.charAt(0).toUpperCase()],
+ )
+}
+
+async function seedWorkspace(companyId: string, ownerId = OWNER_ID): Promise<void> {
+ await seedUser(ownerId)
+ await seedCompany(companyId, ownerId)
+ await seedMember(companyId, ownerId, 'owner')
+}
+
+async function createDocWithImage(
+ companyId: string,
+ docId: string,
+ creatorId: string,
+ imageKey: string,
+): Promise<void> {
+ await pool.query(
+ `INSERT INTO documents (id, company_id, title, created_by)
+ VALUES ($1, $2, 'Test Doc', $3)`,
+ [docId, companyId, creatorId],
+ )
+ const doc = new Y.Doc()
+ const fragment = doc.getXmlFragment('default')
+ const nodes = markdownToYXml(`Here is an image:\n\n![embedded](/${imageKey})`)
+ fragment.push(nodes)
+ const update = Y.encodeStateAsUpdate(doc)
+ await pool.query(
+ `INSERT INTO document_updates (document_id, author_id, update_bytes)
+ VALUES ($1, $2, $3)`,
+ [docId, creatorId, Buffer.from(update)],
+ )
+}
+
+function companyHeaders(companyId: string): Record<string, string> {
+ return { 'content-type': 'application/json', 'x-company-id': companyId }
+}
+
+test('[integration] workspace deletion collects document image keys and purges them on cleanup', async () => {
+ const companyId = 'co-doc-cleanup'
+ const altCompanyId = 'co-doc-alt'
+ await seedWorkspace(companyId, OWNER_ID)
+ await seedWorkspace(altCompanyId, OWNER_ID) // needed so deleting companyId is not the user's only workspace
+
+ const docId = 'doc-with-image'
+ const imageKey = 'attachments/ws-embedded.png'
+ await createDocWithImage(companyId, docId, OWNER_ID, imageKey)
+
+ const deletedObjects: string[] = []
+ const res = await fetch(`${ownerBase}/api/companies/${companyId}`, {
+ method: 'DELETE',
+ headers: companyHeaders(companyId),
+ body: JSON.stringify({ confirmation: `Workspace ${companyId}` }),
+ })
+ assert.equal(res.status, 200, await res.text())
+
+ const jobs = await pool.query<{ storage_keys: string[] }>(
+ `SELECT storage_keys FROM workspace_cleanup_jobs WHERE company_id = $1`,
+ [companyId],
+ )
+ assert.equal(jobs.rowCount, 1)
+ assert.ok(jobs.rows[0].storage_keys.includes(imageKey), `storage_keys should include ${imageKey}`)
+
+ const drainResult = await drainWorkspaceCleanupJobs({
+ dependencies: {
+ deleteStorageObject: async (key) => {
+ deletedObjects.push(key)
+ return true
+ },
+ },
+ })
+ assert.equal(drainResult.completed, 1)
+ assert.ok(deletedObjects.includes(imageKey), `storage object ${imageKey} should be deleted`)
+})
+
+test('[integration] single document deletion enqueues embedded images for cleanup', async () => {
+ const companyId = 'co-single-doc'
+ await seedWorkspace(companyId, OWNER_ID)
+
+ const docId = 'doc-single'
+ const imageKey = 'attachments/single-embedded.png'
+ await createDocWithImage(companyId, docId, OWNER_ID, imageKey)
+
+ const deletedObjects: string[] = []
+ const res = await fetch(`${ownerBase}/api/documents/${docId}`, {
+ method: 'DELETE',
+ headers: companyHeaders(companyId),
+ })
+ assert.equal(res.status, 200, await res.text())
+
+ const jobs = await pool.query<{ storage_keys: string[] }>(
+ `SELECT storage_keys FROM workspace_cleanup_jobs WHERE company_id = $1`,
+ [companyId],
+ )
+ assert.equal(jobs.rowCount, 1)
+ assert.ok(jobs.rows[0].storage_keys.includes(imageKey), `storage_keys should include ${imageKey}`)
+
+ const drainResult = await drainWorkspaceCleanupJobs({
+ dependencies: {
+ deleteStorageObject: async (key) => {
+ deletedObjects.push(key)
+ return true
+ },
+ },
+ })
+ assert.equal(drainResult.completed, 1)
+ assert.ok(deletedObjects.includes(imageKey), `storage object ${imageKey} should be deleted`)
+})
+
+test('[integration] findReferencedStorageKeys protects shared images until last document is deleted', async () => {
+ const companyId = 'co-shared-doc'
+ await seedWorkspace(companyId, OWNER_ID)
+
+ const docA = 'doc-shared-a'
+ const docB = 'doc-shared-b'
+ const sharedKey = 'attachments/shared-between-docs.png'
+ await createDocWithImage(companyId, docA, OWNER_ID, sharedKey)
+ await createDocWithImage(companyId, docB, OWNER_ID, sharedKey)
+
+ // Both docs reference sharedKey
+ const initialRef = await findReferencedStorageKeys([sharedKey])
+ assert.ok(initialRef.has(sharedKey), 'should be referenced by docs')
+
+ // Delete Doc A
+ const resA = await fetch(`${ownerBase}/api/documents/${docA}`, {
+ method: 'DELETE',
+ headers: companyHeaders(companyId),
+ })
+ assert.equal(resA.status, 200)
+
+ // Doc B is still alive, so sharedKey should be protected from deletion
+ const deletedObjects: string[] = []
+ const drain1 = await drainWorkspaceCleanupJobs({
+ dependencies: {
+ deleteStorageObject: async (key) => {
+ deletedObjects.push(key)
+ return true
+ },
+ },
+ })
+ assert.equal(drain1.completed, 1)
+ assert.equal(deletedObjects.includes(sharedKey), false, 'sharedKey must NOT be deleted while docB exists')
+
+ // Now delete Doc B
+ const resB = await fetch(`${ownerBase}/api/documents/${docB}`, {
+ method: 'DELETE',
+ headers: companyHeaders(companyId),
+ })
+ assert.equal(resB.status, 200)
+
+ // Now no surviving docs reference sharedKey, so it should be deleted
+ const drain2 = await drainWorkspaceCleanupJobs({
+ dependencies: {
+ deleteStorageObject: async (key) => {
+ deletedObjects.push(key)
+ return true
+ },
+ },
+ })
+ assert.equal(drain2.completed, 1)
+ assert.ok(deletedObjects.includes(sharedKey), 'sharedKey should be deleted after docB is deleted')
+})
+
+test('[integration] CLI doc delete cleans up embedded document images', async () => {
+ const { companyId, agentId } = await seedCompanyWithAgent()
+ const docId = 'doc-cli-delete'
+ const imageKey = 'attachments/cli-deleted-doc.png'
+ await createDocWithImage(companyId, docId, agentId, imageKey)
+
+ const del = await runCli(['--as', agentId, 'doc', 'delete', docId])
+ assert.equal(del.ok, true, `doc delete failed: ${del.text}`)
+
+ const jobs = await pool.query<{ storage_keys: string[] }>(
+ `SELECT storage_keys FROM workspace_cleanup_jobs WHERE company_id = $1`,
+ [companyId],
+ )
+ assert.equal(jobs.rowCount, 1)
+ assert.ok(jobs.rows[0].storage_keys.includes(imageKey), `storage_keys should include ${imageKey}`)
+
+ const deletedObjects: string[] = []
+ const drainResult = await drainWorkspaceCleanupJobs({
+ dependencies: {
+ deleteStorageObject: async (key) => {
+ deletedObjects.push(key)
+ return true
+ },
+ },
+ })
+ assert.equal(drainResult.completed, 1)
+ assert.ok(deletedObjects.includes(imageKey), `storage object ${imageKey} should be deleted`)
+})
diff --git a/server/src/__tests__/document-image-cleanup.test.ts b/server/src/__tests__/document-image-cleanup.test.ts
new file mode 100644
index 0000000..48f367e
--- /dev/null
+++ b/server/src/__tests__/document-image-cleanup.test.ts
@@ -0,0 +1,82 @@
+import { test, after } from 'node:test'
+import assert from 'node:assert/strict'
+import * as Y from 'yjs'
+import { extractStorageKeysFromDoc } from '../documents/rooms.js'
+import { markdownToYXml, proseMirrorNodeToYXml } from '../documents/markdown.js'
+import { teardownAll } from '../__integration__/_helpers.js'
+
+after(async () => {
+ await teardownAll()
+})
+
+test('extractStorageKeysFromDoc extracts native image nodes with storageKey or src', () => {
+ const doc = new Y.Doc()
+ const fragment = doc.getXmlFragment('default')
+
+ const img1 = proseMirrorNodeToYXml({
+ type: 'image',
+ attrs: {
+ src: 'https://cdn.cumora.ai/attachments/image-1.png',
+ storageKey: 'attachments/image-1.png',
+ alt: 'first',
+ },
+ })
+ const img2 = proseMirrorNodeToYXml({
+ type: 'image',
+ attrs: {
+ src: '/uploads/attachments/image-2.png',
+ alt: 'second',
+ },
+ })
+ const externalImg = proseMirrorNodeToYXml({
+ type: 'image',
+ attrs: {
+ src: 'https://external-site.com/avatar.jpg',
+ alt: 'external',
+ },
+ })
+
+ fragment.push([img1, img2, externalImg])
+
+ const keys = extractStorageKeysFromDoc(doc)
+ assert.deepEqual(keys.sort(), ['attachments/image-1.png', 'attachments/image-2.png'])
+})
+
+test('extractStorageKeysFromDoc extracts keys from markdown image paragraphs and delta links', () => {
+ const doc = new Y.Doc()
+ const fragment = doc.getXmlFragment('default')
+
+ const nodes = markdownToYXml('![chart](/uploads/attachments/chart.png)\n\n[Download PDF](attachments/document.pdf)')
+ fragment.push(nodes)
+
+ const keys = extractStorageKeysFromDoc(doc)
+ assert.deepEqual(keys.sort(), [
+ 'attachments/chart.png',
+ 'attachments/document.pdf',
+ ])
+})
+
+test('extractStorageKeysFromDoc deduplicates repeated storage keys', () => {
+ const doc = new Y.Doc()
+ const fragment = doc.getXmlFragment('default')
+
+ const img1 = proseMirrorNodeToYXml({
+ type: 'image',
+ attrs: {
+ storageKey: 'attachments/repeat.png',
+ src: '/uploads/attachments/repeat.png',
+ },
+ })
+ const img2 = proseMirrorNodeToYXml({
+ type: 'image',
+ attrs: {
+ storageKey: 'attachments/repeat.png',
+ src: '/uploads/attachments/repeat.png',
+ },
+ })
+
+ fragment.push([img1, img2])
+
+ const keys = extractStorageKeysFromDoc(doc)
+ assert.deepEqual(keys, ['attachments/repeat.png'])
+})
diff --git a/server/src/agents/cli.ts b/server/src/agents/cli.ts
index e5cfe11..ebcbf4b 100644
--- a/server/src/agents/cli.ts
+++ b/server/src/agents/cli.ts
@@ -6342,6 +6342,14 @@ async function cmdDocAsActiveAgent(
if (result.imagesDeleted === 0) {
return err(`no images in ${docId} matched the criterion`)
}
+ if (result.deletedStorageKeys && result.deletedStorageKeys.length > 0) {
+ const { enqueueWorkspaceCleanup, nudgeWorkspaceCleanupWorker } = await import('../workspace-cleanup.js')
+ await enqueueWorkspaceCleanup(dbClient, { companyId, agentIds: [], storageKeys: result.deletedStorageKeys })
+ onFinally(() => {
+ nudgeWorkspaceCleanupWorker()
+ return Promise.resolve()
+ })
+ }
await onChanged('document.updated', docId)
return ok(`deleted ${result.imagesDeleted} image${result.imagesDeleted === 1 ? '' : 's'} from ${docId}`, [{
event: 'document.updated',
@@ -6438,7 +6446,18 @@ async function cmdDocAsActiveAgent(
)
if (rows.length === 0) return err(`document ${docId} not found`)
if (rows[0].created_by !== me) return err(`only the creator can delete document ${docId}`)
+ const { collectDocumentStorageKeys, evictDocumentRoom } = await import('../documents/rooms.js')
+ const storageKeys = await collectDocumentStorageKeys(docId, dbClient)
await dbClient.query(`DELETE FROM documents WHERE id = $1`, [docId])
+ if (storageKeys.length > 0) {
+ const { enqueueWorkspaceCleanup, nudgeWorkspaceCleanupWorker } = await import('../workspace-cleanup.js')
+ await enqueueWorkspaceCleanup(dbClient, { companyId, agentIds: [], storageKeys })
+ onFinally(() => {
+ nudgeWorkspaceCleanupWorker()
+ return Promise.resolve()
+ })
+ }
+ evictDocumentRoom(docId)
await onChanged('document.deleted', docId)
return ok(`deleted document ${docId}`, [{
event: 'document.deleted',
diff --git a/server/src/api/router.ts b/server/src/api/router.ts
index 23eb1a9..066630a 100644
--- a/server/src/api/router.ts
+++ b/server/src/api/router.ts
@@ -8,6 +8,7 @@ import { pool } from '../db/pool.js'
import { CH_MESSAGE_NEW, CH_REACTIONS, CH_CONVO_UPDATED, CH_DOCS, CH_TYPING, CH_CALENDAR_EVENTS, CH_BOARDS, CH_STATUS, CH_WORKSPACES, publish } from '../redis.js'
import { enqueueBroadcast, nudgeRealtimeOutbox, withOutboxTransaction } from '../realtime-outbox.js'
import { enqueueWorkspaceCleanup, nudgeWorkspaceCleanupWorker } from '../workspace-cleanup.js'
+import { collectDocumentStorageKeys, evictDocumentRoom } from '../documents/rooms.js'
import { createPoll, castVote, closePoll, PollError } from '../polls.js'
import { env } from '../env.js'
import { publicBodyParserError } from '../body-parser-errors.js'
@@ -1809,6 +1810,14 @@ api.delete('/companies/:id', safe(async (req, res) => {
if (key) storageKeys.add(key)
}
}
+ const { rows: docRows } = await client.query<{ id: string }>(
+ `SELECT id FROM documents WHERE company_id = $1`, [companyId],
+ )
+ for (const docRow of docRows) {
+ const docKeys = await collectDocumentStorageKeys(docRow.id, client)
+ for (const key of docKeys) storageKeys.add(key)
+ evictDocumentRoom(docRow.id)
+ }
// Child/root rows with soft company_id references. FK-backed trees such as
// boards, calendar, projects, invitations, and shipping cascade from the
@@ -7051,10 +7060,19 @@ api.delete('/documents/:id', safe(async (req, res) => {
const role = roleRows[0]?.role ?? 'member'
if (!PRIVILEGED_ROLES.has(role)) throw new HttpError(403, 'only the creator or an owner can delete')
}
+ let docKeys: string[] = []
await withOutboxTransaction(async (client) => {
+ docKeys = await collectDocumentStorageKeys(id, client)
await client.query(`DELETE FROM documents WHERE id = $1`, [id])
+ if (docKeys.length > 0) {
+ await enqueueWorkspaceCleanup(client, { companyId, agentIds: [], storageKeys: docKeys })
+ }
await enqueueDocumentChanged(client, companyId, id, 'document.deleted', userId)
})
+ evictDocumentRoom(id)
+ if (docKeys.length > 0) {
+ nudgeWorkspaceCleanupWorker()
+ }
res.json({ ok: true })
}))
diff --git a/server/src/documents/rooms.ts b/server/src/documents/rooms.ts
index c033742..fefce9e 100644
--- a/server/src/documents/rooms.ts
+++ b/server/src/documents/rooms.ts
@@ -517,6 +517,103 @@ function xmlAttrString(el: Y.XmlElement, key: string): string {
return typeof value === 'string' ? value : ''
}
+/** Pure in-memory extraction of all storage keys referenced in a document.
+ * Checks native image nodes, markdown image paragraphs, link marks, and
+ * embedded attachment references without triggering any URL refresh or DB write. */
+export function extractStorageKeysFromDoc(doc: Y.Doc): string[] {
+ const keys = new Set<string>()
+ const fragment = pmFragment(doc)
+
+ function inspectString(val: string) {
+ if (!val) return
+ const direct = normalizeStorageKey(val) ?? storageKeyFromPublicUrl(val)
+ if (direct) {
+ keys.add(direct)
+ return
+ }
+ const matches = val.matchAll(/(?:attachments|email-attachments|avatars)\/[A-Za-z0-9_.-]+/g)
+ for (const m of matches) {
+ const k = normalizeStorageKey(m[0])
+ if (k) keys.add(k)
+ }
+ }
+
+ function walk(parent: Y.XmlFragment | Y.XmlElement) {
+ for (let i = 0; i < parent.length; i++) {
+ const child = parent.get(i) as Y.AbstractType<unknown>
+ if (child instanceof Y.XmlElement) {
+ if (child.nodeName === 'image') {
+ const key = imageStorageKey(child)
+ if (key) {
+ keys.add(key)
+ } else {
+ inspectString(xmlAttrString(child, 'src'))
+ }
+ } else if (child.nodeName === 'paragraph') {
+ const replacement = paragraphImageNode(child)
+ if (replacement && 'attrs' in replacement && replacement.attrs?.src) {
+ inspectString(String(replacement.attrs.src))
+ }
+ }
+ walk(child)
+ } else if (child instanceof Y.XmlText) {
+ const delta = child.toDelta() as Array<{ insert?: unknown; attributes?: unknown }>
+ for (const op of delta) {
+ const href = hrefFromDeltaAttributes(op.attributes)
+ if (href) inspectString(href)
+ if (typeof op.insert === 'string' && (op.insert.includes('attachments/') || op.insert.includes('avatars/'))) {
+ inspectString(op.insert)
+ }
+ }
+ }
+ }
+ }
+
+ walk(fragment)
+ return Array.from(keys)
+}
+
+/** Collect storage keys for a document. Reuses an active in-memory room if present,
+ * otherwise hydrates a temporary in-memory Y.Doc without adding to the room map,
+ * registering listeners, or refreshing presigned URLs. */
+export async function collectDocumentStorageKeys(
+ documentId: string,
+ dbClient?: PoolClient,
+): Promise<string[]> {
+ const existing = rooms.get(roomKey(documentId))
+ if (existing) {
+ try {
+ await existing.loaded
+ return extractStorageKeysFromDoc(existing.doc)
+ } catch {
+ // Fall through to cold hydration if existing room threw
+ }
+ }
+
+ const doc = new Y.Doc()
+ try {
+ await hydrateDoc(documentId, doc, dbClient)
+ return extractStorageKeysFromDoc(doc)
+ } finally {
+ doc.destroy()
+ }
+}
+
+/** Evict an in-memory document room immediately, e.g. upon document deletion. */
+export function evictDocumentRoom(documentId: string): void {
+ const pending = evictions.get(documentId)
+ if (pending) {
+ clearTimeout(pending)
+ evictions.delete(documentId)
+ }
+ const room = rooms.get(roomKey(documentId))
+ if (room) {
+ rooms.delete(roomKey(documentId))
+ room.subs.clear()
+ room.doc.destroy()
+ }
+}
+
function escapeMarkdownImageText(text: string): string {
return text.replace(/\\/g, '\\\\').replace(/]/g, '\\]')
}
@@ -735,13 +832,20 @@ export async function applyAgentEdit(
| { kind: 'imageDelete'; match: AgentImageDeleteMatch }
>,
dbClient?: PoolClient,
-): Promise<{ replaced: number; imagePlaced: 'absolute' | 'anchor' | 'anchor-missed' | null; imagesDeleted: number; blocksReplaced: number }> {
+): Promise<{
+ replaced: number
+ imagePlaced: 'absolute' | 'anchor' | 'anchor-missed' | null
+ imagesDeleted: number
+ blocksReplaced: number
+ deletedStorageKeys: string[]
+}> {
const room = await getOrCreateRoom(documentId, companyId, dbClient)
const fragment = pmFragment(room.doc)
let replaced = 0
let imagePlaced: 'absolute' | 'anchor' | 'anchor-missed' | null = null
let imagesDeleted = 0
let blocksReplaced = 0
+ const deletedStorageKeys: string[] = []
const origin = { originId: `agent:${agentId}`, authorId: agentId } as never
room.doc.transact(() => {
for (const op of ops) {
@@ -825,6 +929,8 @@ export async function applyAgentEdit(
// images but possible).
matches.sort((a, b) => b.index - a.index)
for (const m of matches) {
+ const key = imageStorageKey(m.element)
+ if (key) deletedStorageKeys.push(key)
m.container.delete(m.index, 1)
imagesDeleted++
}
@@ -843,7 +949,7 @@ export async function applyAgentEdit(
}
normalizeMarkdownImageParagraphChildren(fragment)
}, origin)
- return { replaced, imagePlaced, imagesDeleted, blocksReplaced }
+ return { replaced, imagePlaced, imagesDeleted, blocksReplaced, deletedStorageKeys }
}
/** Cross-instance bus bootstrap. Idempotent — safe to call from index.ts
diff --git a/server/src/workspace-cleanup.ts b/server/src/workspace-cleanup.ts
index 5a49bc9..62a3490 100644
--- a/server/src/workspace-cleanup.ts
+++ b/server/src/workspace-cleanup.ts
@@ -8,6 +8,7 @@ import {
storage,
storageKeyFromPublicUrl,
} from './storage.js'
+import { collectDocumentStorageKeys } from './documents/rooms.js'
interface CleanupJob {
id: string
@@ -74,16 +75,18 @@ async function claimBatch(limit: number): Promise<CleanupJob[]> {
return rows
}
-async function findReferencedStorageKeys(keys: string[]): Promise<Set<string>> {
- if (keys.length === 0) return new Set()
+export async function findReferencedStorageKeys(keys: string[], client?: PoolClient): Promise<Set<string>> {
+ const uniqueKeys = Array.from(new Set(keys.map(normalizeStorageKey).filter((k): k is string => Boolean(k))))
+ if (uniqueKeys.length === 0) return new Set()
+ const db = client ?? pool
const referenced = new Set<string>()
- const [emailFiles, messageFiles, avatars] = await Promise.all([
- pool.query<{ storage_key: string }>(
+ const [emailFiles, messageFiles, avatars, docCandidates] = await Promise.all([
+ db.query<{ storage_key: string }>(
`SELECT storage_key FROM email_attachments WHERE storage_key = ANY($1::text[])`,
- [keys],
+ [uniqueKeys],
),
- pool.query<{ attachment: unknown }>(
+ db.query<{ attachment: unknown }>(
`SELECT attachment
FROM messages m
WHERE attachment IS NOT NULL
@@ -91,9 +94,9 @@ async function findReferencedStorageKeys(keys: string[]): Promise<Set<string>> {
SELECT 1 FROM unnest($1::text[]) candidate(key)
WHERE m.attachment::text LIKE '%' || candidate.key || '%'
)`,
- [keys],
+ [uniqueKeys],
),
- pool.query<{ avatar_url: string }>(
+ db.query<{ avatar_url: string }>(
`SELECT avatar_url
FROM participants p
WHERE avatar_url IS NOT NULL
@@ -101,26 +104,47 @@ async function findReferencedStorageKeys(keys: string[]): Promise<Set<string>> {
SELECT 1 FROM unnest($1::text[]) candidate(key)
WHERE p.avatar_url LIKE '%' || candidate.key || '%'
)`,
- [keys],
+ [uniqueKeys],
+ ),
+ db.query<{ document_id: string }>(
+ `SELECT DISTINCT document_id FROM (
+ SELECT document_id
+ FROM document_updates u, unnest($1::text[]) candidate(key)
+ WHERE position(convert_to(candidate.key, 'UTF8') in u.update_bytes) > 0
+ UNION ALL
+ SELECT document_id
+ FROM document_snapshots s, unnest($1::text[]) candidate(key)
+ WHERE position(convert_to(candidate.key, 'UTF8') in s.state_bytes) > 0
+ ) candidates
+ WHERE document_id IN (SELECT id FROM documents)`,
+ [uniqueKeys],
),
])
for (const row of emailFiles.rows) {
const key = normalizeStorageKey(row.storage_key)
- if (key && keys.includes(key)) referenced.add(key)
+ if (key && uniqueKeys.includes(key)) referenced.add(key)
}
for (const row of messageFiles.rows) {
const attachments = Array.isArray(row.attachment) ? row.attachment : [row.attachment]
for (const attachment of attachments) {
if (!attachment || typeof attachment !== 'object') continue
const key = messageAttachmentStorageKey(attachment as { key?: unknown; url?: unknown })
- if (key && keys.includes(key)) referenced.add(key)
+ if (key && uniqueKeys.includes(key)) referenced.add(key)
}
}
for (const row of avatars.rows) {
const key = storageKeyFromPublicUrl(row.avatar_url)
- if (key && keys.includes(key)) referenced.add(key)
+ if (key && uniqueKeys.includes(key)) referenced.add(key)
}
+ await Promise.all(
+ docCandidates.rows.map(async (row) => {
+ const docKeys = await collectDocumentStorageKeys(row.document_id, client)
+ for (const key of docKeys) {
+ if (uniqueKeys.includes(key)) referenced.add(key)
+ }
+ }),
+ )
return referenced
}