Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Improve post-workspace cleanup cron job (#3296)

Kiran K49ac4fa31b9a
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-customers.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-customers.ts
new file mode 100644
index 0000000000..a238fd6a2c
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-customers.ts
@@ -0,0 +1,64 @@
+import { isStored, storage } from "@/lib/storage";
+import { prisma } from "@dub/prisma";
+import { R2_URL } from "@dub/utils";
+import {
+ DeleteWorkspacePayload,
+ enqueueNextWorkspaceDeleteStep,
+} from "./utils";
+
+const MAX_CUSTOMERS_PER_BATCH = 100;
+
+export async function deleteWorkspaceCustomers(
+ payload: DeleteWorkspacePayload,
+) {
+ const { workspaceId, startingAfter } = payload;
+
+ const customers = await prisma.customer.findMany({
+ where: {
+ projectId: workspaceId,
+ },
+ orderBy: {
+ id: "asc",
+ },
+ ...(startingAfter && {
+ skip: 1,
+ cursor: {
+ id: startingAfter,
+ },
+ }),
+ take: MAX_CUSTOMERS_PER_BATCH,
+ });
+
+ if (customers.length > 0) {
+ // Delete customer avatars from storage
+ await Promise.allSettled(
+ customers.map(async (customer) => {
+ if (customer.avatar && isStored(customer.avatar)) {
+ await storage.delete({
+ key: customer.avatar.replace(`${R2_URL}/`, ""),
+ });
+ }
+ }),
+ );
+
+ const deletedCustomers = await prisma.customer.deleteMany({
+ where: {
+ id: {
+ in: customers.map(({ id }) => id),
+ },
+ },
+ });
+
+ console.log(
+ `Deleted ${deletedCustomers.count} customers for workspace ${workspaceId}.`,
+ );
+ }
+
+ return await enqueueNextWorkspaceDeleteStep({
+ payload,
+ currentStep: "delete-customers",
+ nextStep: "delete-workspace",
+ items: customers,
+ maxBatchSize: MAX_CUSTOMERS_PER_BATCH,
+ });
+}
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-domains.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-domains.ts
new file mode 100644
index 0000000000..04e3186721
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-domains.ts
@@ -0,0 +1,54 @@
+import { removeDomainFromVercel } from "@/lib/api/domains/remove-domain-vercel";
+import { prisma } from "@dub/prisma";
+import {
+ DeleteWorkspacePayload,
+ enqueueNextWorkspaceDeleteStep,
+} from "./utils";
+
+const MAX_DOMAINS_PER_BATCH = 10;
+
+export async function deleteWorkspaceDomains(payload: DeleteWorkspacePayload) {
+ const { workspaceId, startingAfter } = payload;
+
+ const domains = await prisma.domain.findMany({
+ where: {
+ projectId: workspaceId,
+ },
+ orderBy: {
+ id: "asc",
+ },
+ ...(startingAfter && {
+ skip: 1,
+ cursor: {
+ id: startingAfter,
+ },
+ }),
+ take: MAX_DOMAINS_PER_BATCH,
+ });
+
+ if (domains.length > 0) {
+ const deletedDomains = await prisma.domain.deleteMany({
+ where: {
+ id: {
+ in: domains.map(({ id }) => id),
+ },
+ },
+ });
+
+ console.log(
+ `Deleted ${deletedDomains.count} domains for workspace ${workspaceId}.`,
+ );
+
+ await Promise.allSettled(
+ domains.map(({ slug }) => removeDomainFromVercel(slug)),
+ );
+ }
+
+ return await enqueueNextWorkspaceDeleteStep({
+ payload,
+ currentStep: "delete-domains",
+ nextStep: "delete-folders",
+ items: domains,
+ maxBatchSize: MAX_DOMAINS_PER_BATCH,
+ });
+}
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-folders.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-folders.ts
new file mode 100644
index 0000000000..f725313551
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-folders.ts
@@ -0,0 +1,49 @@
+import { prisma } from "@dub/prisma";
+import {
+ DeleteWorkspacePayload,
+ enqueueNextWorkspaceDeleteStep,
+} from "./utils";
+
+const MAX_FOLDERS_PER_BATCH = 100;
+
+export async function deleteWorkspaceFolders(payload: DeleteWorkspacePayload) {
+ const { workspaceId, startingAfter } = payload;
+
+ const folders = await prisma.folder.findMany({
+ where: {
+ projectId: workspaceId,
+ },
+ orderBy: {
+ id: "asc",
+ },
+ ...(startingAfter && {
+ skip: 1,
+ cursor: {
+ id: startingAfter,
+ },
+ }),
+ take: MAX_FOLDERS_PER_BATCH,
+ });
+
+ if (folders.length > 0) {
+ const deletedFolders = await prisma.folder.deleteMany({
+ where: {
+ id: {
+ in: folders.map(({ id }) => id),
+ },
+ },
+ });
+
+ console.log(
+ `Deleted ${deletedFolders.count} folders for workspace ${workspaceId}.`,
+ );
+ }
+
+ return await enqueueNextWorkspaceDeleteStep({
+ payload,
+ currentStep: "delete-folders",
+ nextStep: "delete-customers",
+ items: folders,
+ maxBatchSize: MAX_FOLDERS_PER_BATCH,
+ });
+}
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-links.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-links.ts
new file mode 100644
index 0000000000..ccda2d00df
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-links.ts
@@ -0,0 +1,52 @@
+import { bulkDeleteLinks } from "@/lib/api/links/bulk-delete-links";
+import { prisma } from "@dub/prisma";
+import {
+ DeleteWorkspacePayload,
+ enqueueNextWorkspaceDeleteStep,
+} from "./utils";
+
+const MAX_LINKS_PER_BATCH = 100;
+
+export async function deleteWorkspaceLinks(payload: DeleteWorkspacePayload) {
+ const { workspaceId, startingAfter } = payload;
+
+ const links = await prisma.link.findMany({
+ where: {
+ projectId: workspaceId,
+ },
+ orderBy: {
+ id: "asc",
+ },
+ ...(startingAfter && {
+ skip: 1,
+ cursor: {
+ id: startingAfter,
+ },
+ }),
+ take: MAX_LINKS_PER_BATCH,
+ });
+
+ if (links.length > 0) {
+ const deletedLinks = await prisma.link.deleteMany({
+ where: {
+ id: {
+ in: links.map(({ id }) => id),
+ },
+ },
+ });
+
+ console.log(
+ `Deleted ${deletedLinks.count} links for workspace ${workspaceId}.`,
+ );
+
+ await bulkDeleteLinks(links);
+ }
+
+ return await enqueueNextWorkspaceDeleteStep({
+ payload,
+ currentStep: "delete-links",
+ nextStep: "delete-domains",
+ items: links,
+ maxBatchSize: MAX_LINKS_PER_BATCH,
+ });
+}
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace.ts
new file mode 100644
index 0000000000..a24e3766a4
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace.ts
@@ -0,0 +1,28 @@
+import { prisma } from "@dub/prisma";
+import { logAndRespond } from "../../utils";
+import { DeleteWorkspacePayload } from "./utils";
+
+export async function deleteWorkspace(payload: DeleteWorkspacePayload) {
+ const { workspaceId } = payload;
+
+ const workspace = await prisma.project.findUnique({
+ where: {
+ id: workspaceId,
+ },
+ select: {
+ id: true,
+ },
+ });
+
+ if (!workspace) {
+ return logAndRespond(`Workspace ${workspaceId} not found. Skipping...`);
+ }
+
+ await prisma.project.delete({
+ where: {
+ id: workspaceId,
+ },
+ });
+
+ return logAndRespond(`Workspace ${workspaceId} deleted successfully.`);
+}
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/route.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/route.ts
index ac68abe863..3aaf4cd907 100644
--- a/apps/web/app/(ee)/api/cron/workspaces/delete/route.ts
+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/route.ts
@@ -1,111 +1,30 @@
-import { removeDomainFromVercel } from "@/lib/api/domains/remove-domain-vercel";
-import { handleAndReturnErrorResponse } from "@/lib/api/errors";
-import { bulkDeleteLinks } from "@/lib/api/links/bulk-delete-links";
-import { queueWorkspaceDeletion } from "@/lib/api/workspaces/delete-workspace";
-import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
-import { prisma } from "@dub/prisma";
-import { z } from "zod";
+import { withCron } from "@/lib/cron/with-cron";
+import { logAndRespond } from "../../utils";
+import { deleteWorkspace } from "./delete-workspace";
+import { deleteWorkspaceCustomers } from "./delete-workspace-customers";
+import { deleteWorkspaceDomains } from "./delete-workspace-domains";
+import { deleteWorkspaceFolders } from "./delete-workspace-folders";
+import { deleteWorkspaceLinks } from "./delete-workspace-links";
+import { deleteWorkspaceSchema } from "./utils";
export const dynamic = "force-dynamic";
-const schema = z.object({
- workspaceId: z.string(),
-});
-
// POST /api/cron/workspaces/delete
-export async function POST(req: Request) {
- try {
- const rawBody = await req.text();
- await verifyQstashSignature({ req, rawBody });
-
- const { workspaceId } = schema.parse(JSON.parse(rawBody));
-
- const workspace = await prisma.project.findUnique({
- where: {
- id: workspaceId,
- },
- });
-
- if (!workspace) {
- return new Response(`Workspace ${workspaceId} not found. Skipping...`);
- }
-
- // Delete links in batches
- const links = await prisma.link.findMany({
- where: {
- projectId: workspace.id,
- },
- include: {
- tags: {
- select: {
- tag: true,
- },
- },
- },
- take: 100, // TODO: We can adjust this number based on the performance
- });
-
- if (links.length > 0) {
- const res = await Promise.all([
- prisma.link.deleteMany({
- where: {
- id: {
- in: links.map((link) => link.id),
- },
- },
- }),
-
- bulkDeleteLinks(links),
- ]);
-
- console.log(res);
- }
-
- const remainingLinks = await prisma.link.count({
- where: {
- projectId: workspace.id,
- },
- });
-
- if (remainingLinks > 0) {
- await queueWorkspaceDeletion({
- workspaceId: workspace.id,
- delay: 2,
- });
-
- return new Response(
- `Deleted ${links.length} links, ${remainingLinks} remaining. Starting next batch...`,
- );
- }
-
- // Delete the custom domains
- const domains = await prisma.domain.findMany({
- where: {
- projectId: workspace.id,
- },
- });
-
- if (domains.length > 0) {
- await Promise.all([
- prisma.domain.deleteMany({
- where: {
- projectId: workspace.id,
- },
- }),
-
- domains.map(({ slug }) => removeDomainFromVercel(slug)),
- ]);
- }
-
- // Delete the workspace
- await prisma.project.delete({
- where: { id: workspace.id },
- });
-
- return new Response(
- `Deleted ${links.length} links, no more links remaining. Workspace deleted.`,
- );
- } catch (error) {
- return handleAndReturnErrorResponse(error);
+export const POST = withCron(async ({ rawBody }) => {
+ const payload = deleteWorkspaceSchema.parse(JSON.parse(rawBody));
+
+ switch (payload.step) {
+ case "delete-links":
+ return await deleteWorkspaceLinks(payload);
+ case "delete-domains":
+ return await deleteWorkspaceDomains(payload);
+ case "delete-folders":
+ return await deleteWorkspaceFolders(payload);
+ case "delete-customers":
+ return await deleteWorkspaceCustomers(payload);
+ case "delete-workspace":
+ return await deleteWorkspace(payload);
+ default:
+ return logAndRespond(`Unknown step ${payload.step}`);
}
-}
+});
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/utils.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/utils.ts
new file mode 100644
index 0000000000..99e5abb3c2
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/utils.ts
@@ -0,0 +1,52 @@
+import { qstash } from "@/lib/cron";
+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
+import { z } from "zod";
+import { logAndRespond } from "../../utils";
+
+export const deleteWorkspaceSchema = z.object({
+ workspaceId: z.string(),
+ step: z
+ .enum([
+ "delete-links",
+ "delete-domains",
+ "delete-folders",
+ "delete-customers",
+ "delete-workspace",
+ ])
+ .optional()
+ .default("delete-links"),
+ startingAfter: z.string().optional(),
+});
+
+export type DeleteWorkspacePayload = z.infer<typeof deleteWorkspaceSchema>;
+
+export async function enqueueNextWorkspaceDeleteStep({
+ payload,
+ currentStep,
+ nextStep,
+ items,
+ maxBatchSize,
+}: {
+ payload: DeleteWorkspacePayload;
+ currentStep: DeleteWorkspacePayload["step"];
+ nextStep: DeleteWorkspacePayload["step"];
+ items: { id: string }[];
+ maxBatchSize: number;
+}) {
+ const hasMore = items.length === maxBatchSize;
+
+ const { messageId } = await qstash.publishJSON({
+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,
+ body: {
+ ...payload,
+ startingAfter: hasMore ? items[items.length - 1].id : undefined,
+ step: hasMore ? currentStep : nextStep,
+ },
+ });
+
+ return logAndRespond(
+ hasMore
+ ? `Enqueued next batch for step "${currentStep}" (messageId: ${messageId})`
+ : `Completed step "${currentStep}", moving to "${nextStep}" (messageId: ${messageId})`,
+ );
+}
diff --git a/apps/web/lib/api/workspaces/delete-workspace.ts b/apps/web/lib/api/workspaces/delete-workspace.ts
index 7fbb8784c0..fb2f6a166d 100644
--- a/apps/web/lib/api/workspaces/delete-workspace.ts
+++ b/apps/web/lib/api/workspaces/delete-workspace.ts
@@ -6,6 +6,7 @@ import {
DUB_DOMAINS_ARRAY,
LEGAL_USER_ID,
LEGAL_WORKSPACE_ID,
+ prettyPrint,
R2_URL,
} from "@dub/utils";
import { waitUntil } from "@vercel/functions";
@@ -34,10 +35,12 @@ export async function deleteWorkspace(
defaultWorkspace: null,
},
}),
- ]);
+ ]).then((results) => {
+ console.log(prettyPrint(results));
+ });
waitUntil(
- Promise.all([
+ Promise.allSettled([
// Remove the API keys
prisma.restrictedToken.deleteMany({
where: {
@@ -45,7 +48,7 @@ export async function deleteWorkspace(
},
}),
- // Cancel the workspace's Stripe subscription
+ // Cancel the workspace's Stripe subscription if exists
workspace.stripeId && cancelSubscription(workspace.stripeId),
// Delete workspace logo if it's a custom logo stored in R2
@@ -53,11 +56,16 @@ export async function deleteWorkspace(
workspace.logo.startsWith(`${R2_URL}/logos/${workspace.id}`) &&
storage.delete({ key: workspace.logo.replace(`${R2_URL}/`, "") }),
- // queue the workspace for deletion
- queueWorkspaceDeletion({
- workspaceId: workspace.id,
+ // Queue the workspace for deletion
+ qstash.publishJSON({
+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,
+ body: {
+ workspaceId: workspace.id,
+ },
}),
- ]),
+ ]).then((results) => {
+ console.log(prettyPrint(results));
+ }),
);
}
@@ -129,6 +137,28 @@ export async function deleteWorkspaceAdmin(
deleteDomainsLinksResponse,
);
+ // Delete folders
+ const deleteFoldersResponse = await prisma.folder.deleteMany({
+ where: {
+ projectId: workspace.id,
+ },
+ });
+
+ console.log(
+ `Deleted ${deleteFoldersResponse.count} folders for ${workspace.slug}`,
+ );
+
+ // Delete customers
+ const deleteCustomersResponse = await prisma.customer.deleteMany({
+ where: {
+ projectId: workspace.id,
+ },
+ });
+
+ console.log(
+ `Deleted ${deleteCustomersResponse.count} customers for ${workspace.slug}`,
+ );
+
const deleteWorkspaceResponse = await Promise.allSettled([
// delete workspace logo if it's a custom logo stored in R2
workspace.logo &&
@@ -151,19 +181,3 @@ export async function deleteWorkspaceAdmin(
deleteWorkspaceResponse,
};
}
-
-export async function queueWorkspaceDeletion({
- workspaceId,
- delay,
-}: {
- workspaceId: string;
- delay?: number;
-}) {
- return await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,
- ...(delay && { delay }),
- body: {
- workspaceId,
- },
- });
-}
diff --git a/apps/web/scripts/find-workspaces-without-users.ts b/apps/web/scripts/find-workspaces-without-users.ts
new file mode 100644
index 0000000000..4c069be9a1
--- /dev/null
+++ b/apps/web/scripts/find-workspaces-without-users.ts
@@ -0,0 +1,65 @@
+import { qstash } from "@/lib/cron";
+import { prisma } from "@dub/prisma";
+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
+import "dotenv-flow/config";
+
+async function main() {
+ // Find all projects that don't have any ProjectUsers
+ const workspacesWithoutUsers = await prisma.project.findMany({
+ where: {
+ users: {
+ none: {},
+ },
+ plan: "free",
+ },
+ select: {
+ id: true,
+ name: true,
+ slug: true,
+ plan: true,
+ createdAt: true,
+ updatedAt: true,
+ },
+ orderBy: {
+ createdAt: "desc",
+ },
+ take: 25, // to avoid overwhelming the queue
+ });
+
+ console.log(`Found ${workspacesWithoutUsers.length} projects without users.`);
+
+ if (workspacesWithoutUsers.length === 0) {
+ return;
+ }
+
+ console.table(
+ workspacesWithoutUsers.map((project) => ({
+ id: project.id,
+ name: project.name,
+ slug: project.slug,
+ plan: project.plan,
+ createdAt: project.createdAt.toISOString(),
+ })),
+ );
+
+ const delayIncrement = 30;
+
+ for (let i = 0; i < workspacesWithoutUsers.length; i++) {
+ const project = workspacesWithoutUsers[i];
+ const delay = delayIncrement * (i + 1);
+
+ const response = await qstash.publishJSON({
+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,
+ body: {
+ workspaceId: project.id,
+ },
+ delay,
+ });
+
+ console.log(
+ `Queued deletion for project ${project.id} (${project.slug}) with ${delay}s delay (messageId: ${response.messageId})`,
+ );
+ }
+}
+
+main();
diff --git a/packages/prisma/schema/customer.prisma b/packages/prisma/schema/customer.prisma
index 1b160f17e4..4c119699d3 100644
--- a/packages/prisma/schema/customer.prisma
+++ b/packages/prisma/schema/customer.prisma
@@ -23,7 +23,7 @@ model Customer {
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
- project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+ project Project @relation(fields: [projectId], references: [id])
link Link? @relation(fields: [linkId], references: [id])
program Program? @relation(fields: [programId], references: [id], onDelete: Cascade)
partner Partner? @relation(fields: [partnerId], references: [id], onDelete: Cascade)
diff --git a/packages/prisma/schema/folder.prisma b/packages/prisma/schema/folder.prisma
index 2199bdd525..0cbeb32bc0 100644
--- a/packages/prisma/schema/folder.prisma
+++ b/packages/prisma/schema/folder.prisma
@@ -24,7 +24,7 @@ model Folder {
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
- project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+ project Project @relation(fields: [projectId], references: [id])
links Link[]
users FolderUser[]
accessRequests FolderAccessRequest[]
diff --git a/packages/prisma/schema/misc.prisma b/packages/prisma/schema/misc.prisma
index 40120a5682..9cc0f7d920 100644
--- a/packages/prisma/schema/misc.prisma
+++ b/packages/prisma/schema/misc.prisma
@@ -8,7 +8,7 @@ model YearInReview {
topCountries Json
workspaceId String
- workspace Project @relation(fields: [workspaceId], references: [id])
+ workspace Project @relation(fields: [workspaceId], references: [id], onDelete: Cascade)
createdAt DateTime @default(now())
sentAt DateTime?