Skip to content

armgit / kaneo

Public Imported from GitHub
0

⚡ Project management that doesn't become the project. Boards, Gantt charts, and GitHub integration. Self-host it or use Kaneo Cloud.

fix(storage): schedule cleanup fairly and protect finalized objects

Tinb0ae087fd00b
diff --git a/apps/api/src/project/controllers/delete-project.ts b/apps/api/src/project/controllers/delete-project.ts
index d751a2823..e8e9a1642 100644
--- a/apps/api/src/project/controllers/delete-project.ts
+++ b/apps/api/src/project/controllers/delete-project.ts
@@ -3,10 +3,7 @@ import { HTTPException } from "hono/http-exception";
import db from "../../database";
import { projectTable, assetTable } from "../../database/schema";
import { publishEvent } from "../../events";
-import {
- queueStorageCleanup,
- retryStorageCleanup,
-} from "../../storage/cleanup-queue";
+import { queueStorageCleanup } from "../../storage/cleanup-queue";
import { getProjectSubtaskParentProjects } from "../../task/get-subtask-parent-projects";
import getProject from "./get-project";
@@ -51,7 +48,6 @@ async function deleteProject(id: string, workspaceId: string) {
await publishEvent("subtask-parents.refresh", {
projects: parents.filter((parent) => parent.projectId !== id),
});
- await retryStorageCleanup().catch(() => {});
return existingProject;
}
diff --git a/apps/api/src/storage/cleanup-queue.ts b/apps/api/src/storage/cleanup-queue.ts
index cd0607e24..f92606eab 100644
--- a/apps/api/src/storage/cleanup-queue.ts
+++ b/apps/api/src/storage/cleanup-queue.ts
@@ -1,6 +1,10 @@
import { eq, sql } from "drizzle-orm";
import db from "../database";
-import { storageCleanupTable } from "../database/schema";
+import {
+ assetTable,
+ projectTable,
+ storageCleanupTable,
+} from "../database/schema";
import { deleteS3Object } from "./s3";
export async function queueStorageCleanup(
@@ -20,25 +24,71 @@ export async function retryStorageCleanup(): Promise<{ degraded: boolean }> {
.select()
.from(storageCleanupTable)
.orderBy(
- sql`${storageCleanupTable.lastAttemptAt} nulls first`,
- storageCleanupTable.createdAt,
+ sql`coalesce(${storageCleanupTable.lastAttemptAt}, ${storageCleanupTable.createdAt})`,
+ storageCleanupTable.objectKey,
)
.limit(100);
let degraded = false;
for (const item of pending) {
- try {
- await deleteS3Object(item.objectKey);
- await db
- .delete(storageCleanupTable)
- .where(eq(storageCleanupTable.objectKey, item.objectKey));
- } catch {
- // Keep the durable key for the next tick, without logging uploaded paths.
- degraded = true;
- await db
- .update(storageCleanupTable)
- .set({ lastAttemptAt: new Date() })
- .where(eq(storageCleanupTable.objectKey, item.objectKey));
- }
+ await withStorageObject(item.objectKey, async (tx) => {
+ const queued = await tx.query.storageCleanupTable.findFirst({
+ where: eq(storageCleanupTable.objectKey, item.objectKey),
+ });
+ if (!queued) return;
+ const asset = await tx.query.assetTable.findFirst({
+ columns: { id: true },
+ where: eq(assetTable.objectKey, item.objectKey),
+ });
+ const background = await tx.query.projectTable.findFirst({
+ columns: { id: true },
+ where: eq(projectTable.backgroundObjectKey, item.objectKey),
+ });
+ if (asset || background) {
+ await tx
+ .delete(storageCleanupTable)
+ .where(eq(storageCleanupTable.objectKey, item.objectKey));
+ return;
+ }
+ try {
+ await deleteS3Object(item.objectKey);
+ await tx
+ .delete(storageCleanupTable)
+ .where(eq(storageCleanupTable.objectKey, item.objectKey));
+ } catch (error) {
+ degraded = true;
+ // Provider identifiers diagnose failures without exposing uploaded paths.
+ const failure = error as {
+ name?: string;
+ code?: string;
+ $metadata?: { httpStatusCode?: number };
+ } | null;
+ console.error("Storage cleanup failed", {
+ name: failure?.name,
+ code: failure?.code,
+ status: failure?.$metadata?.httpStatusCode,
+ });
+ await tx
+ .update(storageCleanupTable)
+ .set({ lastAttemptAt: new Date() })
+ .where(eq(storageCleanupTable.objectKey, item.objectKey));
+ }
+ });
}
return { degraded };
}
+
+// Coordinate finalization and deletion of this object only. Verification runs
+// inside this lock so a finalizer waiting behind cleanup sees the missing object.
+export async function withStorageObject<T>(
+ objectKey: string,
+ apply: (
+ tx: Parameters<Parameters<typeof db.transaction>[0]>[0],
+ ) => Promise<T>,
+): Promise<T> {
+ return db.transaction(async (tx) => {
+ await tx.execute(
+ sql`SELECT pg_advisory_xact_lock(hashtext('storage-object'), hashtext(${objectKey}))`,
+ );
+ return apply(tx);
+ });
+}
diff --git a/apps/api/src/task/controllers/bulk-update-tasks.ts b/apps/api/src/task/controllers/bulk-update-tasks.ts
index 4f9e6490f..c40693fab 100644
--- a/apps/api/src/task/controllers/bulk-update-tasks.ts
+++ b/apps/api/src/task/controllers/bulk-update-tasks.ts
@@ -1,7 +1,4 @@
-import {
- queueStorageCleanup,
- retryStorageCleanup,
-} from "../../storage/cleanup-queue";
+import { queueStorageCleanup } from "../../storage/cleanup-queue";
import { assetTable } from "../../database/schema";
import { and, eq, inArray } from "drizzle-orm";
import { HTTPException } from "hono/http-exception";
@@ -246,7 +243,6 @@ async function bulkUpdateTasks({
);
return tx.delete(taskTable).where(inArray(taskTable.id, foundIds));
});
- await retryStorageCleanup().catch(() => {});
updatedCount = result.rowCount ?? foundIds.length;
diff --git a/apps/api/src/task/controllers/delete-task.ts b/apps/api/src/task/controllers/delete-task.ts
index 8f6c3d214..d6db4ce54 100644
--- a/apps/api/src/task/controllers/delete-task.ts
+++ b/apps/api/src/task/controllers/delete-task.ts
@@ -7,10 +7,7 @@ import {
taskTable,
} from "../../database/schema";
import { publishEvent } from "../../events";
-import {
- queueStorageCleanup,
- retryStorageCleanup,
-} from "../../storage/cleanup-queue";
+import { queueStorageCleanup } from "../../storage/cleanup-queue";
import getTask from "./get-task";
async function deleteTask(taskId: string, currentUserId: string) {
@@ -72,8 +69,6 @@ async function deleteTask(taskId: string, currentUserId: string) {
});
}
- await retryStorageCleanup().catch(() => {});
-
return task;
}
diff --git a/apps/api/src/task/index.ts b/apps/api/src/task/index.ts
index 53c2e04e9..a9b1ce15c 100644
--- a/apps/api/src/task/index.ts
+++ b/apps/api/src/task/index.ts
@@ -1,3 +1,4 @@
+import { withStorageObject } from "../storage/cleanup-queue";
import { eq } from "drizzle-orm";
import { HTTPException } from "hono/http-exception";
import { requireEntitlement } from "../billing/require-entitlement-middleware";
@@ -1004,69 +1005,73 @@ const task = apiRouter<BaseVariables & { workspaceId: string }>()
});
}
- let uploaded: Awaited<ReturnType<typeof verifyTaskAssetUpload>>;
- try {
- uploaded = await verifyTaskAssetUpload(normalizedKey, {
- size,
- contentType,
- });
- } catch (error) {
- throw new HTTPException(
- error instanceof InvalidUploadedAssetError ? 400 : 503,
- {
- message:
- error instanceof InvalidUploadedAssetError
- ? error.message
- : "Unable to verify uploaded object.",
- },
- );
- }
-
- const [existingAsset] = await db
- .select({ id: assetTable.id })
- .from(assetTable)
- .where(eq(assetTable.objectKey, normalizedKey))
- .limit(1);
-
- const [asset] = existingAsset
- ? await db
- .update(assetTable)
- .set({
- workspaceId: taskContext.workspaceId,
- projectId: taskContext.projectId,
- taskId: taskContext.taskId,
- filename,
- mimeType: uploaded.contentType,
- size: uploaded.size,
- kind: isImageContentType(uploaded.contentType)
- ? "image"
- : "attachment",
- surface,
- createdBy: userId || null,
- })
- .where(eq(assetTable.id, existingAsset.id))
- .returning({
- id: assetTable.id,
- })
- : await db
- .insert(assetTable)
- .values({
- workspaceId: taskContext.workspaceId,
- projectId: taskContext.projectId,
- taskId: taskContext.taskId,
- objectKey: normalizedKey,
- filename,
- mimeType: uploaded.contentType,
- size: uploaded.size,
- kind: isImageContentType(uploaded.contentType)
- ? "image"
- : "attachment",
- surface,
- createdBy: userId || null,
- })
- .returning({
- id: assetTable.id,
- });
+ const asset = await withStorageObject(normalizedKey, async (db) => {
+ let uploaded: Awaited<ReturnType<typeof verifyTaskAssetUpload>>;
+ try {
+ uploaded = await verifyTaskAssetUpload(normalizedKey, {
+ size,
+ contentType,
+ });
+ } catch (error) {
+ throw new HTTPException(
+ error instanceof InvalidUploadedAssetError ? 400 : 503,
+ {
+ message:
+ error instanceof InvalidUploadedAssetError
+ ? error.message
+ : "Unable to verify uploaded object.",
+ },
+ );
+ }
+
+ const [existingAsset] = await db
+ .select({ id: assetTable.id })
+ .from(assetTable)
+ .where(eq(assetTable.objectKey, normalizedKey))
+ .limit(1);
+
+ const [saved] = existingAsset
+ ? await db
+ .update(assetTable)
+ .set({
+ workspaceId: taskContext.workspaceId,
+ projectId: taskContext.projectId,
+ taskId: taskContext.taskId,
+ filename,
+ mimeType: uploaded.contentType,
+ size: uploaded.size,
+ kind: isImageContentType(uploaded.contentType)
+ ? "image"
+ : "attachment",
+ surface,
+ createdBy: userId || null,
+ })
+ .where(eq(assetTable.id, existingAsset.id))
+ .returning({
+ id: assetTable.id,
+ })
+ : await db
+ .insert(assetTable)
+ .values({
+ workspaceId: taskContext.workspaceId,
+ projectId: taskContext.projectId,
+ taskId: taskContext.taskId,
+ objectKey: normalizedKey,
+ filename,
+ mimeType: uploaded.contentType,
+ size: uploaded.size,
+ kind: isImageContentType(uploaded.contentType)
+ ? "image"
+ : "attachment",
+ surface,
+ createdBy: userId || null,
+ })
+ .returning({
+ id: assetTable.id,
+ });
+
+ return saved;
+ });
if (!asset) {
throw new HTTPException(500, {
diff --git a/tests/api-integration/project-storage-cleanup.test.ts b/tests/api-integration/project-storage-cleanup.test.ts
index 6dbf4f745..8633f0e70 100644
--- a/tests/api-integration/project-storage-cleanup.test.ts
+++ b/tests/api-integration/project-storage-cleanup.test.ts
@@ -1,3 +1,5 @@
+import deleteTask from "../../apps/api/src/task/controllers/delete-task";
+import bulkUpdateTasks from "../../apps/api/src/task/controllers/bulk-update-tasks";
import { eq } from "drizzle-orm";
import { beforeEach, expect, it, vi } from "vite-plus/test";
const m = vi.hoisted(() => ({ deleteS3Object: vi.fn() }));
@@ -41,6 +43,9 @@ it("keeps attachment and background keys after cascading project deletion until
.map((row) => row.objectKey)
.sort(),
).toEqual(["synthetic-attachment", "synthetic-background"]);
+ expect(m.deleteS3Object).not.toHaveBeenCalled();
+ await retryStorageCleanup();
+ expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(2);
m.deleteS3Object.mockResolvedValue(undefined);
expect(await retryStorageCleanup()).toEqual({ degraded: false });
expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(0);
@@ -91,3 +96,83 @@ it("does not let a batch of failing objects starve later cleanup", async () => {
),
).not.toContain("synthetic-100");
});
+
+it("retries failed objects even with a continuous backlog of new keys", async () => {
+ await db.insert(schema.storageCleanupTable).values({
+ objectKey: "old-failure",
+ createdAt: new Date(0),
+ lastAttemptAt: new Date(1),
+ });
+ await db
+ .insert(schema.storageCleanupTable)
+ .values(Array.from({ length: 100 }, (_, i) => ({ objectKey: `new-${i}` })));
+ m.deleteS3Object.mockRejectedValue(new Error("offline"));
+ await retryStorageCleanup();
+ expect(m.deleteS3Object).toHaveBeenCalledWith("old-failure");
+});
+it("does not delete an object referenced by a newly finalized live asset", async () => {
+ const { workspace, user } = await createWorkspaceMember();
+ const { project } = await createProjectFixture({ workspaceId: workspace.id });
+ await db
+ .insert(schema.storageCleanupTable)
+ .values({ objectKey: "reused-live-key" });
+ await db.insert(schema.assetTable).values({
+ projectId: project.id,
+ workspaceId: workspace.id,
+ objectKey: "reused-live-key",
+ filename: "test.png",
+ mimeType: "image/png",
+ size: 1,
+ createdBy: user.id,
+ });
+ await retryStorageCleanup();
+ expect(m.deleteS3Object).not.toHaveBeenCalled();
+ expect(await db.select().from(schema.assetTable)).toHaveLength(1);
+});
+
+it.each(["single", "bulk"])(
+ "recovers queued attachments after %s task deletion",
+ async (mode) => {
+ const { workspace, user } = await createWorkspaceMember();
+ const { project } = await createProjectFixture({
+ workspaceId: workspace.id,
+ });
+ const [task] = await db
+ .insert(schema.taskTable)
+ .values({
+ projectId: project.id,
+ title: "delete attachment",
+ status: "to-do",
+ })
+ .returning();
+ await db.insert(schema.assetTable).values({
+ taskId: task.id,
+ projectId: project.id,
+ workspaceId: workspace.id,
+ objectKey: "task-attachment",
+ filename: "test.png",
+ mimeType: "image/png",
+ size: 1,
+ createdBy: user.id,
+ });
+ m.deleteS3Object.mockRejectedValue(new Error("offline"));
+ if (mode === "single") await deleteTask(task.id, user.id);
+ else
+ await bulkUpdateTasks({
+ taskIds: [task.id],
+ operation: "delete",
+ userId: user.id,
+ });
+ expect(m.deleteS3Object).not.toHaveBeenCalled();
+ expect(
+ await db.query.taskTable.findFirst({
+ where: eq(schema.taskTable.id, task.id),
+ }),
+ ).toBeUndefined();
+ await retryStorageCleanup();
+ expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(1);
+ m.deleteS3Object.mockResolvedValue(undefined);
+ await retryStorageCleanup();
+ expect(await db.select().from(schema.storageCleanupTable)).toHaveLength(0);
+ },
+);