⚡ Project management that doesn't become the project. Boards, Gantt charts, and GitHub integration. Self-host it or use Kaneo Cloud.
TypeScript honoissue-managementissue-trackerjira-alternativekanbanlinear-alternativemcpproject-managementreactself-hostedtypescript
fix(storage): schedule cleanup fairly and protect finalized objects
Tinb0ae087fd00b
diff --git a/apps/api/src/project/controllers/delete-project.ts b/apps/api/src/project/controllers/delete-project.tsindex d751a2823..e8e9a1642 100644--- a/apps/api/src/project/controllers/delete-project.ts+++ b/apps/api/src/project/controllers/delete-project.ts@@ -3,10 +3,7 @@ import { HTTPException } from "hono/http-exception";import db from "../../database";import { projectTable, assetTable } from "../../database/schema";import { publishEvent } from "../../events";-import {- queueStorageCleanup,- retryStorageCleanup,-} from "../../storage/cleanup-queue";+import { queueStorageCleanup } from "../../storage/cleanup-queue";import { getProjectSubtaskParentProjects } from "../../task/get-subtask-parent-projects";import getProject from "./get-project";@@ -51,7 +48,6 @@ async function deleteProject(id: string, workspaceId: string) {await publishEvent("subtask-parents.refresh", {projects: parents.filter((parent) => parent.projectId !== id),});- await retryStorageCleanup().catch(() => {});return existingProject;}diff --git a/apps/api/src/storage/cleanup-queue.ts b/apps/api/src/storage/cleanup-queue.tsindex cd0607e24..f92606eab 100644--- a/apps/api/src/storage/cleanup-queue.ts+++ b/apps/api/src/storage/cleanup-queue.ts@@ -1,6 +1,10 @@import { eq, sql } from "drizzle-orm";import db from "../database";-import { storageCleanupTable } from "../database/schema";+import {+ assetTable,+ projectTable,+ storageCleanupTable,+} from "../database/schema";import { deleteS3Object } from "./s3";export async function queueStorageCleanup(@@ -20,25 +24,71 @@ export async function retryStorageCleanup(): Promise<{ degraded: boolean }> {.select().from(storageCleanupTable).orderBy(- sql`${storageCleanupTable.lastAttemptAt} nulls first`,- storageCleanupTable.createdAt,+ sql`coalesce(${storageCleanupTable.lastAttemptAt}, ${storageCleanupTable.createdAt})`,+ storageCleanupTable.objectKey,).limit(100);let degraded = false;for (const item of pending) {- try {- await deleteS3Object(item.objectKey);- await db- .delete(storageCleanupTable)- .where(eq(storageCleanupTable.objectKey, item.objectKey));- } catch {- // Keep the durable key for the next tick, without logging uploaded paths.- degraded = true;- await db- .update(storageCleanupTable)- .set({ lastAttemptAt: new Date() })- .where(eq(storageCleanupTable.objectKey, item.objectKey));- }+ await withStorageObject(item.objectKey, async (tx) => {+ const queued = await tx.query.storageCleanupTable.findFirst({+ where: eq(storageCleanupTable.objectKey, item.objectKey),+ });+ if (!queued) return;+ const asset = await tx.query.assetTable.findFirst({+ columns: { id: true },+ where: eq(assetTable.objectKey, item.objectKey),+ });+ const background = await tx.query.projectTable.findFirst({+ columns: { id: true },+ where: eq(projectTable.backgroundObjectKey, item.objectKey),+ });+ if (asset || background) {+ await tx+ .delete(storageCleanupTable)+ .where(eq(storageCleanupTable.objectKey, item.objectKey));+ return;+ }+ try {+ await deleteS3Object(item.objectKey);+ await tx+ .delete(storageCleanupTable)+ .where(eq(storageCleanupTable.objectKey, item.objectKey));+ } catch (error) {+ degraded = true;+ // Provider identifiers diagnose failures without exposing uploaded paths.+ const failure = error as {+ name?: string;+ code?: string;+ $metadata?: { httpStatusCode?: number };+ } | null;+ console.error("Storage cleanup failed", {+ name: failure?.name,+ code: failure?.code,+ status: failure?.$metadata?.httpStatusCode,+ });+ await tx+ .update(storageCleanupTable)+ .set({ lastAttemptAt: new Date() })+ .where(eq(storageCleanupTable.objectKey, item.objectKey));+ }+ });}return { degraded };}++// Coordinate finalization and deletion of this object only. Verification runs+// inside this lock so a finalizer waiting behind cleanup sees the missing object.+export async function withStorageObject<T>(+ objectKey: string,+ apply: (+ tx: Parameters<Parameters<typeof db.transaction>[0]>[0],+ ) => Promise<T>,+): Promise<T> {+ return db.transaction(async (tx) => {+ await tx.execute(+ sql`SELECT pg_advisory_xact_lock(hashtext('storage-object'), hashtext(${objectKey}))`,+ );+ return apply(tx);+ });+}diff --git a/apps/api/src/task/controllers/bulk-update-tasks.ts b/apps/api/src/task/controllers/bulk-update-tasks.tsindex 4f9e6490f..c40693fab 100644--- a/apps/api/src/task/controllers/bulk-update-tasks.ts+++ b/apps/api/src/task/controllers/bulk-update-tasks.ts@@ -1,7 +1,4 @@-import {- queueStorageCleanup,- retryStorageCleanup,-} from "../../storage/cleanup-queue";+import { queueStorageCleanup } from "../../storage/cleanup-queue";import { assetTable } from "../../database/schema";import { and, eq, inArray } from "drizzle-orm";import { HTTPException } from "hono/http-exception";@@ -246,7 +243,6 @@ async function bulkUpdateTasks({);return tx.delete(taskTable).where(inArray(taskTable.id, foundIds));});- await retryStorageCleanup().catch(() => {});updatedCount = result.rowCount ?? foundIds.length;diff --git a/apps/api/src/task/controllers/delete-task.ts b/apps/api/src/task/controllers/delete-task.tsindex 8f6c3d214..d6db4ce54 100644--- a/apps/api/src/task/controllers/delete-task.ts+++ b/apps/api/src/task/controllers/delete-task.ts@@ -7,10 +7,7 @@ import {taskTable,} from "../../database/schema";import { publishEvent } from "../../events";-import {- queueStorageCleanup,- retryStorageCleanup,-} from "../../storage/cleanup-queue";+import { queueStorageCleanup } from "../../storage/cleanup-queue";import getTask from "./get-task";async function deleteTask(taskId: string, currentUserId: string) {@@ -72,8 +69,6 @@ async function deleteTask(taskId: string, currentUserId: string) {});}- await retryStorageCleanup().catch(() => {});-return task;}diff --git a/apps/api/src/task/index.ts b/apps/api/src/task/index.tsindex 53c2e04e9..a9b1ce15c 100644--- a/apps/api/src/task/index.ts+++ b/apps/api/src/task/index.ts@@ -1,3 +1,4 @@+import { withStorageObject } from "../storage/cleanup-queue";import { eq } from "drizzle-orm";import { HTTPException } from "hono/http-exception";import { requireEntitlement } from "../billing/require-entitlement-middleware";@@ -1004,69 +1005,73 @@ const task = apiRouter<BaseVariables & { workspaceId: string }>()});}- let uploaded: Awaited<ReturnType<typeof verifyTaskAssetUpload>>;- try {- uploaded = await verifyTaskAssetUpload(normalizedKey, {- size,- contentType,- });- } catch (error) {- throw new HTTPException(- error instanceof InvalidUploadedAssetError ? 400 : 503,- {- message:- error instanceof InvalidUploadedAssetError- ? error.message- : "Unable to verify uploaded object.",- },- );- }-- const [existingAsset] = await db- .select({ id: assetTable.id })- .from(assetTable)- .where(eq(assetTable.objectKey, normalizedKey))- .limit(1);-- const [asset] = existingAsset- ? await db- .update(assetTable)- .set({- workspaceId: taskContext.workspaceId,- projectId: taskContext.projectId,- taskId: taskContext.taskId,- filename,- mimeType: uploaded.contentType,- size: uploaded.size,- kind: isImageContentType(uploaded.contentType)- ? "image"- : "attachment",- surface,- createdBy: userId || null,- })- .where(eq(assetTable.id, existingAsset.id))- .returning({- id: assetTable.id,- })- : await db- .insert(assetTable)- .values({- workspaceId: taskContext.workspaceId,- projectId: taskContext.projectId,- taskId: taskContext.taskId,- objectKey: normalizedKey,- filename,- mimeType: uploaded.contentType,- size: uploaded.size,- kind: isImageContentType(uploaded.contentType)- ? "image"- : "attachment",- surface,- createdBy: userId || null,- })- .returning({- id: assetTable.id,- });+ const asset = await withStorageObject(normalizedKey, async (db) => {+ let uploaded: Awaited<ReturnType<typeof verifyTaskAssetUpload>>;+ try {+ uploaded = await verifyTaskAssetUpload(normalizedKey, {+ size,+ contentType,+ });+ } catch (error) {+ throw new HTTPException(+ error instanceof InvalidUploadedAssetError ? 400 : 503,+ {+ message:+ error instanceof InvalidUploadedAssetError+ ? error.message+ : "Unable to verify uploaded object.",+ },+ );+ }++ const [existingAsset] = await db+ .select({ id: assetTable.id })+ .from(assetTable)+ .where(eq(assetTable.objectKey, normalizedKey))+ .limit(1);++ const [saved] = existingAsset+ ? await db+ .update(assetTable)+ .set({+ workspaceId: taskContext.workspaceId,+ projectId: taskContext.projectId,+ taskId: taskContext.taskId,+ filename,+ mimeType: uploaded.contentType,+ size: uploaded.size,+ kind: isImageContentType(uploaded.contentType)+ ? "image"+ : "attachment",+ surface,+ createdBy: userId || null,+ })+ .where(eq(assetTable.id, existingAsset.id))+ .returning({+ id: assetTable.id,+ })+ : await db+ .insert(assetTable)+ .values({+ workspaceId: taskContext.workspaceId,+ projectId: taskContext.projectId,+ taskId: taskContext.taskId,+ objectKey: normalizedKey,+ filename,+ mimeType: uploaded.contentType,+ size: uploaded.size,+ kind: isImageContentType(uploaded.contentType)+ ? "image"+ : "attachment",+ surface,+ createdBy: userId || null,+ })+ .returning({+ id: assetTable.id,+ });++ return saved;+ });if (!asset) {throw new HTTPException(500, {diff --git a/tests/api-integration/project-storage-cleanup.test.ts b/tests/api-integration/project-storage-cleanup.test.tsindex 6dbf4f745..8633f0e70 100644--- a/tests/api-integration/project-storage-cleanup.test.ts+++ b/tests/api-integration/project-storage-cleanup.test.ts@@ -1,3 +1,5 @@+import deleteTask from "../../apps/api/src/task/controllers/delete-task";+import bulkUpdateTasks from "../../apps/api/src/task/controllers/bulk-update-tasks";import { eq } from "drizzle-orm";import { beforeEach, expect, it, vi } from "vite-plus/test";const m = vi.hoisted(() => ({ deleteS3Object: vi.fn() }));@@ -41,6 +43,9 @@ it("keeps attachment and background keys after cascading project deletion until.map((row) => row.objectKey).sort(),).toEqual(["synthetic-attachment", "synthetic-background"]);+ expect(m.deleteS3Object).not.toHaveBeenCalled();+ await retryStorageCleanup();+ expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(2);m.deleteS3Object.mockResolvedValue(undefined);expect(await retryStorageCleanup()).toEqual({ degraded: false });expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(0);@@ -91,3 +96,83 @@ it("does not let a batch of failing objects starve later cleanup", async () => {),).not.toContain("synthetic-100");});++it("retries failed objects even with a continuous backlog of new keys", async () => {+ await db.insert(schema.storageCleanupTable).values({+ objectKey: "old-failure",+ createdAt: new Date(0),+ lastAttemptAt: new Date(1),+ });+ await db+ .insert(schema.storageCleanupTable)+ .values(Array.from({ length: 100 }, (_, i) => ({ objectKey: `new-${i}` })));+ m.deleteS3Object.mockRejectedValue(new Error("offline"));+ await retryStorageCleanup();+ expect(m.deleteS3Object).toHaveBeenCalledWith("old-failure");+});+it("does not delete an object referenced by a newly finalized live asset", async () => {+ const { workspace, user } = await createWorkspaceMember();+ const { project } = await createProjectFixture({ workspaceId: workspace.id });+ await db+ .insert(schema.storageCleanupTable)+ .values({ objectKey: "reused-live-key" });+ await db.insert(schema.assetTable).values({+ projectId: project.id,+ workspaceId: workspace.id,+ objectKey: "reused-live-key",+ filename: "test.png",+ mimeType: "image/png",+ size: 1,+ createdBy: user.id,+ });+ await retryStorageCleanup();+ expect(m.deleteS3Object).not.toHaveBeenCalled();+ expect(await db.select().from(schema.assetTable)).toHaveLength(1);+});++it.each(["single", "bulk"])(+ "recovers queued attachments after %s task deletion",+ async (mode) => {+ const { workspace, user } = await createWorkspaceMember();+ const { project } = await createProjectFixture({+ workspaceId: workspace.id,+ });+ const [task] = await db+ .insert(schema.taskTable)+ .values({+ projectId: project.id,+ title: "delete attachment",+ status: "to-do",+ })+ .returning();+ await db.insert(schema.assetTable).values({+ taskId: task.id,+ projectId: project.id,+ workspaceId: workspace.id,+ objectKey: "task-attachment",+ filename: "test.png",+ mimeType: "image/png",+ size: 1,+ createdBy: user.id,+ });+ m.deleteS3Object.mockRejectedValue(new Error("offline"));+ if (mode === "single") await deleteTask(task.id, user.id);+ else+ await bulkUpdateTasks({+ taskIds: [task.id],+ operation: "delete",+ userId: user.id,+ });+ expect(m.deleteS3Object).not.toHaveBeenCalled();+ expect(+ await db.query.taskTable.findFirst({+ where: eq(schema.taskTable.id, task.id),+ }),+ ).toBeUndefined();+ await retryStorageCleanup();+ expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(1);+ m.deleteS3Object.mockResolvedValue(undefined);+ await retryStorageCleanup();+ expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(0);+ },+);