The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Improve post-workspace cleanup cron job (#3296)
Kiran K49ac4fa31b9a
diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-customers.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-customers.tsnew file mode 100644index 0000000000..a238fd6a2c--- /dev/null+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-customers.ts@@ -0,0 +1,64 @@+import { isStored, storage } from "@/lib/storage";+import { prisma } from "@dub/prisma";+import { R2_URL } from "@dub/utils";+import {+ DeleteWorkspacePayload,+ enqueueNextWorkspaceDeleteStep,+} from "./utils";++const MAX_CUSTOMERS_PER_BATCH = 100;++export async function deleteWorkspaceCustomers(+ payload: DeleteWorkspacePayload,+) {+ const { workspaceId, startingAfter } = payload;++ const customers = await prisma.customer.findMany({+ where: {+ projectId: workspaceId,+ },+ orderBy: {+ id: "asc",+ },+ ...(startingAfter && {+ skip: 1,+ cursor: {+ id: startingAfter,+ },+ }),+ take: MAX_CUSTOMERS_PER_BATCH,+ });++ if (customers.length > 0) {+ // Delete customer avatars from storage+ await Promise.allSettled(+ customers.map(async (customer) => {+ if (customer.avatar && isStored(customer.avatar)) {+ await storage.delete({+ key: customer.avatar.replace(`${R2_URL}/`, ""),+ });+ }+ }),+ );++ const deletedCustomers = await prisma.customer.deleteMany({+ where: {+ id: {+ in: customers.map(({ id }) => id),+ },+ },+ });++ console.log(+ `Deleted ${deletedCustomers.count} customers for workspace ${workspaceId}.`,+ );+ }++ return await enqueueNextWorkspaceDeleteStep({+ payload,+ currentStep: "delete-customers",+ nextStep: "delete-workspace",+ items: customers,+ maxBatchSize: MAX_CUSTOMERS_PER_BATCH,+ });+}diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-domains.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-domains.tsnew file mode 100644index 0000000000..04e3186721--- /dev/null+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-domains.ts@@ -0,0 +1,54 @@+import { removeDomainFromVercel } from "@/lib/api/domains/remove-domain-vercel";+import { prisma } from "@dub/prisma";+import {+ DeleteWorkspacePayload,+ enqueueNextWorkspaceDeleteStep,+} from "./utils";++const MAX_DOMAINS_PER_BATCH = 10;++export async function deleteWorkspaceDomains(payload: DeleteWorkspacePayload) {+ const { workspaceId, startingAfter } = payload;++ const domains = await prisma.domain.findMany({+ where: {+ projectId: workspaceId,+ },+ orderBy: {+ id: "asc",+ },+ ...(startingAfter && {+ skip: 1,+ cursor: {+ id: startingAfter,+ },+ }),+ take: MAX_DOMAINS_PER_BATCH,+ });++ if (domains.length > 0) {+ const deletedDomains = await prisma.domain.deleteMany({+ where: {+ id: {+ in: domains.map(({ id }) => id),+ },+ },+ });++ console.log(+ `Deleted ${deletedDomains.count} domains for workspace ${workspaceId}.`,+ );++ await Promise.allSettled(+ domains.map(({ slug }) => removeDomainFromVercel(slug)),+ );+ }++ return await enqueueNextWorkspaceDeleteStep({+ payload,+ currentStep: "delete-domains",+ nextStep: "delete-folders",+ items: domains,+ maxBatchSize: MAX_DOMAINS_PER_BATCH,+ });+}diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-folders.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-folders.tsnew file mode 100644index 0000000000..f725313551--- /dev/null+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-folders.ts@@ -0,0 +1,49 @@+import { prisma } from "@dub/prisma";+import {+ DeleteWorkspacePayload,+ enqueueNextWorkspaceDeleteStep,+} from "./utils";++const MAX_FOLDERS_PER_BATCH = 100;++export async function deleteWorkspaceFolders(payload: DeleteWorkspacePayload) {+ const { workspaceId, startingAfter } = payload;++ const folders = await prisma.folder.findMany({+ where: {+ projectId: workspaceId,+ },+ orderBy: {+ id: "asc",+ },+ ...(startingAfter && {+ skip: 1,+ cursor: {+ id: startingAfter,+ },+ }),+ take: MAX_FOLDERS_PER_BATCH,+ });++ if (folders.length > 0) {+ const deletedFolders = await prisma.folder.deleteMany({+ where: {+ id: {+ in: folders.map(({ id }) => id),+ },+ },+ });++ console.log(+ `Deleted ${deletedFolders.count} folders for workspace ${workspaceId}.`,+ );+ }++ return await enqueueNextWorkspaceDeleteStep({+ payload,+ currentStep: "delete-folders",+ nextStep: "delete-customers",+ items: folders,+ maxBatchSize: MAX_FOLDERS_PER_BATCH,+ });+}diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-links.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-links.tsnew file mode 100644index 0000000000..ccda2d00df--- /dev/null+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace-links.ts@@ -0,0 +1,52 @@+import { bulkDeleteLinks } from "@/lib/api/links/bulk-delete-links";+import { prisma } from "@dub/prisma";+import {+ DeleteWorkspacePayload,+ enqueueNextWorkspaceDeleteStep,+} from "./utils";++const MAX_LINKS_PER_BATCH = 100;++export async function deleteWorkspaceLinks(payload: DeleteWorkspacePayload) {+ const { workspaceId, startingAfter } = payload;++ const links = await prisma.link.findMany({+ where: {+ projectId: workspaceId,+ },+ orderBy: {+ id: "asc",+ },+ ...(startingAfter && {+ skip: 1,+ cursor: {+ id: startingAfter,+ },+ }),+ take: MAX_LINKS_PER_BATCH,+ });++ if (links.length > 0) {+ const deletedLinks = await prisma.link.deleteMany({+ where: {+ id: {+ in: links.map(({ id }) => id),+ },+ },+ });++ console.log(+ `Deleted ${deletedLinks.count} links for workspace ${workspaceId}.`,+ );++ await bulkDeleteLinks(links);+ }++ return await enqueueNextWorkspaceDeleteStep({+ payload,+ currentStep: "delete-links",+ nextStep: "delete-domains",+ items: links,+ maxBatchSize: MAX_LINKS_PER_BATCH,+ });+}diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace.tsnew file mode 100644index 0000000000..a24e3766a4--- /dev/null+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/delete-workspace.ts@@ -0,0 +1,28 @@+import { prisma } from "@dub/prisma";+import { logAndRespond } from "../../utils";+import { DeleteWorkspacePayload } from "./utils";++export async function deleteWorkspace(payload: DeleteWorkspacePayload) {+ const { workspaceId } = payload;++ const workspace = await prisma.project.findUnique({+ where: {+ id: workspaceId,+ },+ select: {+ id: true,+ },+ });++ if (!workspace) {+ return logAndRespond(`Workspace ${workspaceId} not found. Skipping...`);+ }++ await prisma.project.delete({+ where: {+ id: workspaceId,+ },+ });++ return logAndRespond(`Workspace ${workspaceId} deleted successfully.`);+}diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/route.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/route.tsindex ac68abe863..3aaf4cd907 100644--- a/apps/web/app/(ee)/api/cron/workspaces/delete/route.ts+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/route.ts@@ -1,111 +1,30 @@-import { removeDomainFromVercel } from "@/lib/api/domains/remove-domain-vercel";-import { handleAndReturnErrorResponse } from "@/lib/api/errors";-import { bulkDeleteLinks } from "@/lib/api/links/bulk-delete-links";-import { queueWorkspaceDeletion } from "@/lib/api/workspaces/delete-workspace";-import { verifyQstashSignature } from "@/lib/cron/verify-qstash";-import { prisma } from "@dub/prisma";-import { z } from "zod";+import { withCron } from "@/lib/cron/with-cron";+import { logAndRespond } from "../../utils";+import { deleteWorkspace } from "./delete-workspace";+import { deleteWorkspaceCustomers } from "./delete-workspace-customers";+import { deleteWorkspaceDomains } from "./delete-workspace-domains";+import { deleteWorkspaceFolders } from "./delete-workspace-folders";+import { deleteWorkspaceLinks } from "./delete-workspace-links";+import { deleteWorkspaceSchema } from "./utils";export const dynamic = "force-dynamic";-const schema = z.object({- workspaceId: z.string(),-});-// POST /api/cron/workspaces/delete-export async function POST(req: Request) {- try {- const rawBody = await req.text();- await verifyQstashSignature({ req, rawBody });-- const { workspaceId } = schema.parse(JSON.parse(rawBody));-- const workspace = await prisma.project.findUnique({- where: {- id: workspaceId,- },- });-- if (!workspace) {- return new Response(`Workspace ${workspaceId} not found. Skipping...`);- }-- // Delete links in batches- const links = await prisma.link.findMany({- where: {- projectId: workspace.id,- },- include: {- tags: {- select: {- tag: true,- },- },- },- take: 100, // TODO: We can adjust this number based on the performance- });-- if (links.length > 0) {- const res = await Promise.all([- prisma.link.deleteMany({- where: {- id: {- in: links.map((link) => link.id),- },- },- }),-- bulkDeleteLinks(links),- ]);-- console.log(res);- }-- const remainingLinks = await prisma.link.count({- where: {- projectId: workspace.id,- },- });-- if (remainingLinks > 0) {- await queueWorkspaceDeletion({- workspaceId: workspace.id,- delay: 2,- });-- return new Response(- `Deleted ${links.length} links, ${remainingLinks} remaining. Starting next batch...`,- );- }-- // Delete the custom domains- const domains = await prisma.domain.findMany({- where: {- projectId: workspace.id,- },- });-- if (domains.length > 0) {- await Promise.all([- prisma.domain.deleteMany({- where: {- projectId: workspace.id,- },- }),-- domains.map(({ slug }) => removeDomainFromVercel(slug)),- ]);- }-- // Delete the workspace- await prisma.project.delete({- where: { id: workspace.id },- });-- return new Response(- `Deleted ${links.length} links, no more links remaining. Workspace deleted.`,- );- } catch (error) {- return handleAndReturnErrorResponse(error);+export const POST = withCron(async ({ rawBody }) => {+ const payload = deleteWorkspaceSchema.parse(JSON.parse(rawBody));++ switch (payload.step) {+ case "delete-links":+ return await deleteWorkspaceLinks(payload);+ case "delete-domains":+ return await deleteWorkspaceDomains(payload);+ case "delete-folders":+ return await deleteWorkspaceFolders(payload);+ case "delete-customers":+ return await deleteWorkspaceCustomers(payload);+ case "delete-workspace":+ return await deleteWorkspace(payload);+ default:+ return logAndRespond(`Unknown step ${payload.step}`);}-}+});diff --git a/apps/web/app/(ee)/api/cron/workspaces/delete/utils.ts b/apps/web/app/(ee)/api/cron/workspaces/delete/utils.tsnew file mode 100644index 0000000000..99e5abb3c2--- /dev/null+++ b/apps/web/app/(ee)/api/cron/workspaces/delete/utils.ts@@ -0,0 +1,52 @@+import { qstash } from "@/lib/cron";+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";+import { z } from "zod";+import { logAndRespond } from "../../utils";++export const deleteWorkspaceSchema = z.object({+ workspaceId: z.string(),+ step: z+ .enum([+ "delete-links",+ "delete-domains",+ "delete-folders",+ "delete-customers",+ "delete-workspace",+ ])+ .optional()+ .default("delete-links"),+ startingAfter: z.string().optional(),+});++export type DeleteWorkspacePayload = z.infer<typeof deleteWorkspaceSchema>;++export async function enqueueNextWorkspaceDeleteStep({+ payload,+ currentStep,+ nextStep,+ items,+ maxBatchSize,+}: {+ payload: DeleteWorkspacePayload;+ currentStep: DeleteWorkspacePayload["step"];+ nextStep: DeleteWorkspacePayload["step"];+ items: { id: string }[];+ maxBatchSize: number;+}) {+ const hasMore = items.length === maxBatchSize;++ const { messageId } = await qstash.publishJSON({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,+ body: {+ ...payload,+ startingAfter: hasMore ? items[items.length - 1].id : undefined,+ step: hasMore ? currentStep : nextStep,+ },+ });++ return logAndRespond(+ hasMore+ ? `Enqueued next batch for step "${currentStep}" (messageId: ${messageId})`+ : `Completed step "${currentStep}", moving to "${nextStep}" (messageId: ${messageId})`,+ );+}diff --git a/apps/web/lib/api/workspaces/delete-workspace.ts b/apps/web/lib/api/workspaces/delete-workspace.tsindex 7fbb8784c0..fb2f6a166d 100644--- a/apps/web/lib/api/workspaces/delete-workspace.ts+++ b/apps/web/lib/api/workspaces/delete-workspace.ts@@ -6,6 +6,7 @@ import {DUB_DOMAINS_ARRAY,LEGAL_USER_ID,LEGAL_WORKSPACE_ID,+ prettyPrint,R2_URL,} from "@dub/utils";import { waitUntil } from "@vercel/functions";@@ -34,10 +35,12 @@ export async function deleteWorkspace(defaultWorkspace: null,},}),- ]);+ ]).then((results) => {+ console.log(prettyPrint(results));+ });waitUntil(- Promise.all([+ Promise.allSettled([// Remove the API keysprisma.restrictedToken.deleteMany({where: {@@ -45,7 +48,7 @@ export async function deleteWorkspace(},}),- // Cancel the workspace's Stripe subscription+ // Cancel the workspace's Stripe subscription if existsworkspace.stripeId && cancelSubscription(workspace.stripeId),// Delete workspace logo if it's a custom logo stored in R2@@ -53,11 +56,16 @@ export async function deleteWorkspace(workspace.logo.startsWith(`${R2_URL}/logos/${workspace.id}`) &&storage.delete({ key: workspace.logo.replace(`${R2_URL}/`, "") }),- // queue the workspace for deletion- queueWorkspaceDeletion({- workspaceId: workspace.id,+ // Queue the workspace for deletion+ qstash.publishJSON({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,+ body: {+ workspaceId: workspace.id,+ },}),- ]),+ ]).then((results) => {+ console.log(prettyPrint(results));+ }),);}@@ -129,6 +137,28 @@ export async function deleteWorkspaceAdmin(deleteDomainsLinksResponse,);+ // Delete folders+ const deleteFoldersResponse = await prisma.folder.deleteMany({+ where: {+ projectId: workspace.id,+ },+ });++ console.log(+ `Deleted ${deleteFoldersResponse.count} folders for ${workspace.slug}`,+ );++ // Delete customers+ const deleteCustomersResponse = await prisma.customer.deleteMany({+ where: {+ projectId: workspace.id,+ },+ });++ console.log(+ `Deleted ${deleteCustomersResponse.count} customers for ${workspace.slug}`,+ );+const deleteWorkspaceResponse = await Promise.allSettled([// delete workspace logo if it's a custom logo stored in R2workspace.logo &&@@ -151,19 +181,3 @@ export async function deleteWorkspaceAdmin(deleteWorkspaceResponse,};}--export async function queueWorkspaceDeletion({- workspaceId,- delay,-}: {- workspaceId: string;- delay?: number;-}) {- return await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,- ...(delay && { delay }),- body: {- workspaceId,- },- });-}diff --git a/apps/web/scripts/find-workspaces-without-users.ts b/apps/web/scripts/find-workspaces-without-users.tsnew file mode 100644index 0000000000..4c069be9a1--- /dev/null+++ b/apps/web/scripts/find-workspaces-without-users.ts@@ -0,0 +1,65 @@+import { qstash } from "@/lib/cron";+import { prisma } from "@dub/prisma";+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";+import "dotenv-flow/config";++async function main() {+ // Find all projects that don't have any ProjectUsers+ const workspacesWithoutUsers = await prisma.project.findMany({+ where: {+ users: {+ none: {},+ },+ plan: "free",+ },+ select: {+ id: true,+ name: true,+ slug: true,+ plan: true,+ createdAt: true,+ updatedAt: true,+ },+ orderBy: {+ createdAt: "desc",+ },+ take: 25, // to avoid overwhelming the queue+ });++ console.log(`Found ${workspacesWithoutUsers.length} projects without users.`);++ if (workspacesWithoutUsers.length === 0) {+ return;+ }++ console.table(+ workspacesWithoutUsers.map((project) => ({+ id: project.id,+ name: project.name,+ slug: project.slug,+ plan: project.plan,+ createdAt: project.createdAt.toISOString(),+ })),+ );++ const delayIncrement = 30;++ for (let i = 0; i < workspacesWithoutUsers.length; i++) {+ const project = workspacesWithoutUsers[i];+ const delay = delayIncrement * (i + 1);++ const response = await qstash.publishJSON({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/workspaces/delete`,+ body: {+ workspaceId: project.id,+ },+ delay,+ });++ console.log(+ `Queued deletion for project ${project.id} (${project.slug}) with ${delay}s delay (messageId: ${response.messageId})`,+ );+ }+}++main();diff --git a/packages/prisma/schema/customer.prisma b/packages/prisma/schema/customer.prismaindex 1b160f17e4..4c119699d3 100644--- a/packages/prisma/schema/customer.prisma+++ b/packages/prisma/schema/customer.prisma@@ -23,7 +23,7 @@ model Customer {createdAt DateTime @default(now())updatedAt DateTime @updatedAt- project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)+ project Project @relation(fields: [projectId], references: [id])link Link? @relation(fields: [linkId], references: [id])program Program? @relation(fields: [programId], references: [id], onDelete: Cascade)partner Partner? @relation(fields: [partnerId], references: [id], onDelete: Cascade)diff --git a/packages/prisma/schema/folder.prisma b/packages/prisma/schema/folder.prismaindex 2199bdd525..0cbeb32bc0 100644--- a/packages/prisma/schema/folder.prisma+++ b/packages/prisma/schema/folder.prisma@@ -24,7 +24,7 @@ model Folder {createdAt DateTime @default(now())updatedAt DateTime @updatedAt- project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)+ project Project @relation(fields: [projectId], references: [id])links Link[]users FolderUser[]accessRequests FolderAccessRequest[]diff --git a/packages/prisma/schema/misc.prisma b/packages/prisma/schema/misc.prismaindex 40120a5682..9cc0f7d920 100644--- a/packages/prisma/schema/misc.prisma+++ b/packages/prisma/schema/misc.prisma@@ -8,7 +8,7 @@ model YearInReview {topCountries JsonworkspaceId String- workspace Project @relation(fields: [workspaceId], references: [id])+ workspace Project @relation(fields: [workspaceId], references: [id], onDelete: Cascade)createdAt DateTime @default(now())sentAt DateTime?