Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Add durable background job framework with DB-backed QStash retry (#4130)

Kiran K96f6a39102bc
diff --git a/apps/web/app/(ee)/api/cron/folders/delete/route.ts b/apps/web/app/(ee)/api/cron/folders/delete/route.ts
deleted file mode 100644
index 346a75500e..0000000000
--- a/apps/web/app/(ee)/api/cron/folders/delete/route.ts
+++ /dev/null
@@ -1,95 +0,0 @@
-import { handleAndReturnErrorResponse } from "@/lib/api/errors";
-import { queueFolderDeletion } from "@/lib/api/folders/queue-folder-deletion";
-import { includeProgramEnrollment } from "@/lib/api/links/include-program-enrollment";
-import { includeTags } from "@/lib/api/links/include-tags";
-import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
-import { prisma } from "@/lib/prisma";
-import { recordLink } from "@/lib/tinybird";
-import * as z from "zod/v4";
-
-export const dynamic = "force-dynamic";
-
-const MAX_LINKS_PER_BATCH = 500;
-
-const schema = z.object({
- folderId: z.string(),
-});
-
-// POST /api/cron/folders/delete
-// Recursively remove the `folderId` association in all the links of a folder from Tinybird
-export async function POST(req: Request) {
- try {
- const rawBody = await req.text();
- await verifyQstashSignature({ req, rawBody });
-
- const { folderId } = schema.parse(JSON.parse(rawBody));
-
- const linksToUpdate = await prisma.link.findMany({
- where: {
- folderId,
- },
- take: MAX_LINKS_PER_BATCH,
- orderBy: {
- createdAt: "desc", // TODO we need to add [folderId, createdAt] index on Link table
- },
- include: {
- ...includeTags,
- ...includeProgramEnrollment,
- },
- });
-
- if (linksToUpdate.length === 0) {
- await prisma.projectUsers.updateMany({
- where: {
- defaultFolderId: folderId,
- },
- data: {
- defaultFolderId: null,
- },
- });
-
- await prisma.folder.delete({
- where: {
- id: folderId,
- },
- });
-
- return new Response("No more links to process. Deleting folder...");
- }
-
- const recordLinkResponse = await recordLink(
- linksToUpdate.map((link) => ({
- ...link,
- folderId: null,
- })),
- );
-
- console.log("recordLinkResponse", recordLinkResponse);
-
- const updateLinksResponse = await prisma.link.updateMany({
- where: {
- id: {
- in: linksToUpdate.map((link) => link.id),
- },
- },
- data: {
- folderId: null,
- },
- });
-
- console.log("updateLinksResponse", updateLinksResponse);
-
- // TODO: technically we can check if linksToUpdate.length < MAX_LINKS_PER_BATCH
- // because that means all the links have been updated and we can delete the folder
- await queueFolderDeletion({
- folderId,
- delay: 2,
- });
-
- return new Response(
- `Processed ${linksToUpdate.length} links in the folder.`,
- );
- } catch (error) {
- return handleAndReturnErrorResponse(error);
- }
-}
diff --git a/apps/web/app/(ee)/api/cron/queue/retry/route.ts b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
new file mode 100644
index 0000000000..1c1af54078
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
@@ -0,0 +1,168 @@
+import { logger } from "@/lib/axiom/server";
+import { qstash } from "@/lib/cron";
+import { withCron } from "@/lib/cron/with-cron";
+import { buildReplayRequest, isPublishSuccess } from "@/lib/jobs";
+import { prisma } from "@/lib/prisma";
+import { redis } from "@/lib/upstash/redis";
+import { logAndRespond } from "../../utils";
+
+export const dynamic = "force-dynamic";
+
+const MAX_ATTEMPTS = 10;
+const BATCH_SIZE = 100;
+
+// Lock for the cron job. TTL must be ≥ cron maxDuration (600s in vercel.json)
+// so the lock cannot expire while a run is still alive and allow a concurrent
+// minute-cron invocation
+const LOCK_KEY = "lock:queue-retry";
+const LOCK_TTL_SECONDS = 600;
+
+// GET /api/cron/queue/retry – republish background jobs that failed to
+// publish to QStash at dispatch time; rows are deleted on successful publish
+export const GET = withCron(async () => {
+ const acquired = await redis.set(LOCK_KEY, "1", {
+ nx: true,
+ ex: LOCK_TTL_SECONDS,
+ });
+
+ if (!acquired) {
+ return logAndRespond(
+ "[queue-retry] Another run is in progress. Skipping...",
+ );
+ }
+
+ try {
+ const jobs = await prisma.job.findMany({
+ where: {
+ attempts: {
+ lt: MAX_ATTEMPTS,
+ },
+ },
+ orderBy: {
+ createdAt: "asc",
+ },
+ take: BATCH_SIZE,
+ });
+
+ if (jobs.length === 0) {
+ return logAndRespond("No background jobs to retry.");
+ }
+
+ const now = new Date();
+ const entries = jobs.map((job) => buildReplayRequest(job, now));
+ const jobIds = jobs.map(({ id }) => id);
+ let responses: unknown[];
+
+ try {
+ responses = await qstash.batchJSON(entries);
+ } catch (error) {
+ const errorMessage =
+ error instanceof Error ? error.message : String(error);
+
+ await prisma.job.updateMany({
+ where: {
+ id: {
+ in: jobIds,
+ },
+ },
+ data: {
+ attempts: {
+ increment: 1,
+ },
+ lastError: errorMessage.slice(0, 1000),
+ },
+ });
+
+ logger.error("jobs.retry_failed", {
+ jobCount: jobs.length,
+ errorMessage,
+ });
+
+ // Jobs that just ran out of attempts are excluded from future runs by the
+ // MAX_ATTEMPTS filter and need manual intervention
+ const exhaustedJobs = jobs.filter(
+ (backgroundJob) => backgroundJob.attempts + 1 >= MAX_ATTEMPTS,
+ );
+
+ if (exhaustedJobs.length > 0) {
+ logger.error("jobs.retry_exhausted", {
+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
+ });
+ }
+
+ await logger.flush();
+
+ return logAndRespond(
+ `Failed to republish ${jobs.length} background jobs to QStash.`,
+ );
+ }
+
+ const publishedJobIds: string[] = [];
+ const failedJobs: typeof jobs = [];
+
+ jobs.forEach((job, index) => {
+ if (isPublishSuccess(responses[index])) {
+ publishedJobIds.push(job.id);
+ } else {
+ failedJobs.push(job);
+ }
+ });
+
+ if (publishedJobIds.length > 0) {
+ await prisma.job.deleteMany({
+ where: {
+ id: {
+ in: publishedJobIds,
+ },
+ },
+ });
+ }
+
+ if (failedJobs.length === 0) {
+ return logAndRespond(
+ `Republished ${jobs.length} background jobs to QStash.`,
+ );
+ }
+
+ const errorMessage = "QStash batch publish did not return a messageId";
+
+ await prisma.job.updateMany({
+ where: {
+ id: {
+ in: failedJobs.map(({ id }) => id),
+ },
+ },
+ data: {
+ attempts: {
+ increment: 1,
+ },
+ lastError: errorMessage,
+ },
+ });
+
+ logger.error("jobs.retry_failed", {
+ jobCount: failedJobs.length,
+ errorMessage,
+ });
+
+ // Jobs that just ran out of attempts are excluded from future runs by the
+ // MAX_ATTEMPTS filter and need manual intervention
+ const exhaustedJobs = failedJobs.filter(
+ (backgroundJob) => backgroundJob.attempts + 1 >= MAX_ATTEMPTS,
+ );
+
+ if (exhaustedJobs.length > 0) {
+ logger.error("jobs.retry_exhausted", {
+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
+ });
+ }
+
+ await logger.flush();
+
+ return logAndRespond(
+ `Republished ${publishedJobIds.length} background jobs to QStash; failed to republish ${failedJobs.length}.`,
+ );
+ } finally {
+ await redis.del(LOCK_KEY);
+ }
+});
diff --git a/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts b/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts
index d119666526..c7edc277d1 100644
--- a/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts
+++ b/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts
@@ -20,9 +20,11 @@ const BATCH_SIZE = 6000;
// How many webhook deliveries to enqueue to QStash in parallel
const SEND_CONCURRENCY = 100;
-// Lock for the cron job
+// Lock for the cron job. TTL must be ≥ cron maxDuration (600s in vercel.json)
+// so the lock cannot expire while a run is still alive and allow a concurrent
+// minute-cron invocation
const LOCK_KEY = "lock:send-link-clicked-webhooks";
-const LOCK_TTL_SECONDS = 120;
+const LOCK_TTL_SECONDS = 600;
// Drains the workspace:click:events stream and delivers link.clicked webhooks.
// Runs every minute (see vercel.json).
diff --git a/apps/web/app/(ee)/api/cron/tags/delete-partner-tag/route.ts b/apps/web/app/(ee)/api/cron/tags/delete-partner-tag/route.ts
deleted file mode 100644
index 436d2680a1..0000000000
--- a/apps/web/app/(ee)/api/cron/tags/delete-partner-tag/route.ts
+++ /dev/null
@@ -1,100 +0,0 @@
-import { handleAndReturnErrorResponse } from "@/lib/api/errors";
-import { includeProgramEnrollment } from "@/lib/api/links/include-program-enrollment";
-import { includeTags } from "@/lib/api/links/include-tags";
-import { queuePartnerTagDeletion } from "@/lib/api/tags/queue-tag-deletion";
-import { CRON_BATCH_SIZE } from "@/lib/cron";
-import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
-import { prisma } from "@/lib/prisma";
-import { recordLink } from "@/lib/tinybird";
-import * as z from "zod/v4";
-import { logAndRespond } from "../../utils";
-
-export const dynamic = "force-dynamic";
-
-const schema = z.object({
- id: z.string(),
-});
-
-// POST /api/cron/tags/delete-partner-tag
-export async function POST(req: Request) {
- try {
- const rawBody = await req.text();
- await verifyQstashSignature({ req, rawBody });
-
- const { id: partnerTagId } = schema.parse(JSON.parse(rawBody));
-
- const partnerTag = await prisma.partnerTag.findUnique({
- where: {
- id: partnerTagId,
- },
- });
-
- if (!partnerTag || partnerTag.programId !== null) {
- return logAndRespond(
- `Partner tag ${partnerTagId} not found or not marked for deletion. Skipping...`,
- );
- }
-
- const programPartnerTags = await prisma.programPartnerTag.findMany({
- where: {
- partnerTagId,
- },
- select: {
- id: true,
- programEnrollment: {
- select: {
- id: true,
- },
- },
- },
- take: CRON_BATCH_SIZE,
- });
-
- if (programPartnerTags.length === 0) {
- await prisma.partnerTag.delete({
- where: {
- id: partnerTagId,
- },
- });
-
- return logAndRespond(
- `No ProgramPartnerTag rows left. Partner tag ${partnerTagId} deleted from database.`,
- );
- }
-
- const { count } = await prisma.programPartnerTag.deleteMany({
- where: {
- id: {
- in: programPartnerTags.map((row) => row.id),
- },
- },
- });
- console.log(`Deleted ${count} program–partner tag associations`);
-
- const linksToUpdate = await prisma.link.findMany({
- where: {
- programEnrollment: {
- id: {
- in: programPartnerTags.map((row) => row.programEnrollment.id),
- },
- },
- },
- include: {
- ...includeTags,
- ...includeProgramEnrollment,
- },
- });
-
- console.log(`Found ${linksToUpdate.length} links to update in Tinybird`);
-
- await recordLink(linksToUpdate);
-
- await queuePartnerTagDeletion({
- partnerTagId,
- delay: 2,
- });
- return logAndRespond("Batch completed, starting next batch...");
- } catch (error) {
- return handleAndReturnErrorResponse(error);
- }
-}
diff --git a/apps/web/app/api/folders/[folderId]/route.ts b/apps/web/app/api/folders/[folderId]/route.ts
index ff9913723a..270bcb41ea 100644
--- a/apps/web/app/api/folders/[folderId]/route.ts
+++ b/apps/web/app/api/folders/[folderId]/route.ts
@@ -1,8 +1,8 @@
import { DubApiError } from "@/lib/api/errors";
-import { queueFolderDeletion } from "@/lib/api/folders/queue-folder-deletion";
import { parseRequestBody } from "@/lib/api/utils";
import { withWorkspace } from "@/lib/auth";
import { verifyFolderAccess } from "@/lib/folder/permissions";
+import { folderDeletedJob } from "@/lib/jobs/handlers/folder-deleted-job";
import { getPlanCapabilities } from "@/lib/plan-capabilities";
import { prisma } from "@/lib/prisma";
import { FolderSchema, updateFolderSchema } from "@/lib/zod/schemas/folders";
@@ -127,43 +127,41 @@ export const DELETE = withWorkspace(
where: {
folderId,
},
+ take: 100,
});
- // if there are no links associated with the folder, we can just delete it
- if (linksCount === 0) {
- await prisma.folder.delete({
+ await prisma.$transaction([
+ // If there are no links associated with the folder, we can just delete it
+ ...(linksCount === 0
+ ? [
+ prisma.folder.delete({
+ where: {
+ id: folderId,
+ },
+ }),
+ ]
+ : [
+ prisma.folder.update({
+ where: {
+ id: folderId,
+ },
+ data: {
+ projectId: "",
+ },
+ }),
+ ]),
+
+ // Remove the default folder assignment for all users whose defaultFolderId matches the given folderId
+ prisma.projectUsers.updateMany({
where: {
- id: folderId,
+ defaultFolderId: folderId,
},
- });
- } else {
- await Promise.all([
- prisma.folder.update({
- where: {
- id: folderId,
- },
- data: {
- projectId: "",
- },
- }),
-
- queueFolderDeletion({
- folderId,
- }),
- ]);
- }
-
- // Remove the default folder assignment for all users whose defaultFolderId matches the given folderId
- await prisma.projectUsers.updateMany({
- where: {
- defaultFolderId: folderId,
- },
- data: {
- defaultFolderId: null,
- },
- });
+ data: {
+ defaultFolderId: null,
+ },
+ }),
- waitUntil(
+ // Decrement the folders usage for the workspace
prisma.project.update({
where: {
id: workspace.id,
@@ -174,7 +172,20 @@ export const DELETE = withWorkspace(
},
},
}),
- );
+ ]);
+
+ if (linksCount > 0) {
+ waitUntil(
+ folderDeletedJob.dispatch(
+ {
+ folderId,
+ },
+ {
+ label: folderId,
+ },
+ ),
+ );
+ }
return NextResponse.json({ id: folderId });
},
diff --git a/apps/web/app/api/jobs/process/[jobName]/route.ts b/apps/web/app/api/jobs/process/[jobName]/route.ts
new file mode 100644
index 0000000000..aad8b1f2d9
--- /dev/null
+++ b/apps/web/app/api/jobs/process/[jobName]/route.ts
@@ -0,0 +1,101 @@
+import { logger, withAxiomBodyLog } from "@/lib/axiom/server";
+import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
+import { jobEnvelopeSchema } from "@/lib/jobs";
+import { loadJob } from "@/lib/jobs/registry";
+import * as z from "zod/v4";
+
+export const maxDuration = 600;
+
+// POST /api/jobs/process/[jobName] – shared executor for all background jobs
+export const POST = withAxiomBodyLog(
+ async (
+ req: Request,
+ { params }: { params: Promise<{ jobName: string }> },
+ ) => {
+ const { jobName: urlJobName } = await params;
+ const clonedReq = req.clone();
+ const rawBody = await clonedReq.text();
+
+ await verifyQstashSignature({ req, rawBody });
+
+ let parsedBody: unknown = null;
+
+ try {
+ parsedBody = JSON.parse(rawBody);
+ } catch (error) {
+ // Handled below by the envelope validation
+ }
+
+ const envelope = jobEnvelopeSchema.safeParse(parsedBody);
+
+ // Permanently invalid — returning 2xx stops QStash retries
+ if (!envelope.success) {
+ logger.error("jobs.invalid_envelope", {
+ error: envelope.error.message,
+ });
+ await logger.flush();
+
+ return new Response("Invalid job envelope (non-retryable).");
+ }
+
+ const { name: jobName } = envelope.data;
+
+ if (urlJobName !== jobName) {
+ logger.error("jobs.job_name_mismatch", {
+ urlJobName,
+ envelopeJobName: jobName,
+ });
+ await logger.flush();
+
+ return new Response(
+ `Job name mismatch: URL "${urlJobName}" does not match envelope "${jobName}" (non-retryable).`,
+ );
+ }
+
+ console.log(`[jobs:${jobName}] Executing job...`, envelope.data);
+
+ // Get the job definition
+ const job = await loadJob(jobName);
+
+ if (!job) {
+ logger.error("jobs.unknown_job", {
+ jobName,
+ });
+ await logger.flush();
+
+ return new Response(`Unknown job "${jobName}" (non-retryable).`);
+ }
+
+ try {
+ await job.execute(envelope.data.payload);
+ } catch (error) {
+ if (error instanceof z.ZodError) {
+ // A bad payload is permanent — 200 so QStash doesn't retry
+ logger.error("jobs.invalid_payload", {
+ jobName: job.name,
+ error: error.message,
+ });
+ await logger.flush();
+
+ return new Response(
+ `Invalid payload for job "${job.name}" (non-retryable).`,
+ );
+ }
+
+ // Transient failure → 500 → QStash retries
+ logger.error("jobs.execution_failed", {
+ jobName: job.name,
+ errorName: error instanceof Error ? error.name : undefined,
+ errorMessage: error instanceof Error ? error.message : String(error),
+ errorStack: error instanceof Error ? error.stack : undefined,
+ });
+ await logger.flush();
+
+ return new Response(`Job "${job.name}" failed.`, {
+ status: 500,
+ });
+ }
+
+ return new Response(`Executed job "${job.name}".`);
+ },
+);
diff --git a/apps/web/lib/actions/partners/tags/delete-partner-tag.ts b/apps/web/lib/actions/partners/tags/delete-partner-tag.ts
index 51fb58fa66..adeeb93ee1 100644
--- a/apps/web/lib/actions/partners/tags/delete-partner-tag.ts
+++ b/apps/web/lib/actions/partners/tags/delete-partner-tag.ts
@@ -1,8 +1,10 @@
"use server";
import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw";
-import { markPartnerTagDeleted } from "@/lib/api/tags/mark-partner-tag-deleted";
+import { partnerTagDeletedJob } from "@/lib/jobs/handlers/partner-tag-deleted-job";
+import { prisma } from "@/lib/prisma";
import { deletePartnerTagSchema } from "@/lib/zod/schemas/partner-tags";
+import { waitUntil } from "@vercel/functions";
import { authActionClient } from "../../safe-action";
import { throwIfNoPermission } from "../../throw-if-no-permission";
@@ -20,12 +22,28 @@ export const deletePartnerTagAction = authActionClient
const programId = getDefaultProgramIdOrThrow(workspace);
- const deleted = await markPartnerTagDeleted({
- partnerTagId,
- programId,
+ const { count } = await prisma.partnerTag.updateMany({
+ where: {
+ id: partnerTagId,
+ programId,
+ },
+ data: {
+ programId: null,
+ },
});
- if (!deleted) {
- throw new Error("Partner tag not found.");
+ if (count === 0) {
+ throw new Error("Partner tag not found or already deleted.");
}
+
+ waitUntil(
+ partnerTagDeletedJob.dispatch(
+ {
+ partnerTagId,
+ },
+ {
+ label: partnerTagId,
+ },
+ ),
+ );
});
diff --git a/apps/web/lib/api/create-id.ts b/apps/web/lib/api/create-id.ts
index 3165325eaa..bb4a8b4814 100644
--- a/apps/web/lib/api/create-id.ts
+++ b/apps/web/lib/api/create-id.ts
@@ -47,6 +47,7 @@ const prefixes = [
"sbl_", // submitted lead
"pb_", // partner postback
"req_", // api request log
+ "job_", // background job
] as const;
// ULID uses base32 encoding
diff --git a/apps/web/lib/api/folders/delete-workspace-folders.ts b/apps/web/lib/api/folders/delete-workspace-folders.ts
index f911c9fb80..38249edd3e 100644
--- a/apps/web/lib/api/folders/delete-workspace-folders.ts
+++ b/apps/web/lib/api/folders/delete-workspace-folders.ts
@@ -1,5 +1,5 @@
+import { folderDeletedJob } from "@/lib/jobs/handlers/folder-deleted-job";
import { prisma } from "@/lib/prisma";
-import { queueFolderDeletion } from "./queue-folder-deletion";
export async function deleteWorkspaceFolders({
workspaceId,
@@ -20,6 +20,7 @@ export async function deleteWorkspaceFolders({
});
excludedFolderId = program.defaultFolderId;
}
+
const folders = await prisma.folder.findMany({
where: {
projectId: workspaceId,
@@ -39,22 +40,32 @@ export async function deleteWorkspaceFolders({
return;
}
- await prisma.projectUsers.updateMany({
- where: {
- projectId: workspaceId,
- defaultFolderId: { in: folders.map(({ id }) => id) },
- },
- data: {
- defaultFolderId: null,
- },
- });
+ const folderIds = folders.map(({ id }) => id);
+
+ await prisma.$transaction([
+ prisma.projectUsers.updateMany({
+ where: {
+ projectId: workspaceId,
+ defaultFolderId: {
+ in: folderIds,
+ },
+ },
+ data: {
+ defaultFolderId: null,
+ },
+ }),
+
+ prisma.folder.updateMany({
+ where: {
+ id: {
+ in: folderIds,
+ },
+ },
+ data: {
+ projectId: "",
+ },
+ }),
- return await Promise.all([
- ...folders.map(({ id }) =>
- queueFolderDeletion({
- folderId: id,
- }),
- ),
prisma.project.update({
where: {
id: workspaceId,
@@ -64,4 +75,11 @@ export async function deleteWorkspaceFolders({
},
}),
]);
+
+ await folderDeletedJob.dispatchBatch(
+ folderIds.map((folderId) => ({ folderId })),
+ ({ folderId }) => ({
+ label: folderId,
+ }),
+ );
}
diff --git a/apps/web/lib/api/folders/queue-folder-deletion.ts b/apps/web/lib/api/folders/queue-folder-deletion.ts
deleted file mode 100644
index 0bcaa4ff5b..0000000000
--- a/apps/web/lib/api/folders/queue-folder-deletion.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import { qstash } from "@/lib/cron";
-import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
-
-export async function queueFolderDeletion({
- folderId,
- delay,
-}: {
- folderId: string;
- delay?: number;
-}) {
- return await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/folders/delete`,
- ...(delay && { delay }),
- body: {
- folderId,
- },
- });
-}
diff --git a/apps/web/lib/api/tags/mark-partner-tag-deleted.ts b/apps/web/lib/api/tags/mark-partner-tag-deleted.ts
deleted file mode 100644
index ca067083d9..0000000000
--- a/apps/web/lib/api/tags/mark-partner-tag-deleted.ts
+++ /dev/null
@@ -1,41 +0,0 @@
-import { prisma } from "@/lib/prisma";
-import { queuePartnerTagDeletion } from "./queue-tag-deletion";
-
-// Mark the partner tag as deleted (orphan from program). ProgramPartnerTag
-// rows and the PartnerTag row are removed asynchronously via a cron job.
-export async function markPartnerTagDeleted({
- partnerTagId,
- programId,
-}: {
- partnerTagId: string;
- programId: string;
-}): Promise<boolean> {
- try {
- const updated = await prisma.partnerTag.update({
- where: {
- id: partnerTagId,
- programId,
- },
- data: {
- programId: null,
- },
- });
-
- if (!updated) {
- return false;
- }
-
- await queuePartnerTagDeletion({
- partnerTagId,
- });
-
- return true;
- } catch (error) {
- console.error("markPartnerTagDeleted", {
- reason: error,
- partnerTagId,
- programId,
- });
- return false;
- }
-}
diff --git a/apps/web/lib/api/tags/queue-tag-deletion.ts b/apps/web/lib/api/tags/queue-tag-deletion.ts
index 78a62db2bb..6e9af100df 100644
--- a/apps/web/lib/api/tags/queue-tag-deletion.ts
+++ b/apps/web/lib/api/tags/queue-tag-deletion.ts
@@ -16,19 +16,3 @@ export async function queueLinkTagDeletion({
},
});
}
-
-export async function queuePartnerTagDeletion({
- partnerTagId,
- delay,
-}: {
- partnerTagId: string;
- delay?: number;
-}) {
- return await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/tags/delete-partner-tag`,
- ...(delay && { delay }),
- body: {
- id: partnerTagId,
- },
- });
-}
diff --git a/apps/web/lib/jobs/handlers/folder-deleted-job.ts b/apps/web/lib/jobs/handlers/folder-deleted-job.ts
new file mode 100644
index 0000000000..83d4464b37
--- /dev/null
+++ b/apps/web/lib/jobs/handlers/folder-deleted-job.ts
@@ -0,0 +1,104 @@
+import * as z from "zod/v4";
+import { includeProgramEnrollment } from "../../api/links/include-program-enrollment";
+import { includeTags } from "../../api/links/include-tags";
+import { prisma } from "../../prisma";
+import { recordLink } from "../../tinybird";
+import { defineJob } from "../index";
+
+const MAX_LINKS_PER_BATCH = 500;
+
+const inputSchema = z.object({
+ folderId: z.string(),
+});
+
+// Job to remove folderId from all links and delete the folder
+export const folderDeletedJob = defineJob({
+ name: "folder-deleted-job",
+ schema: inputSchema,
+ async handle(input) {
+ const { folderId } = input;
+
+ const folder = await prisma.folder.findUnique({
+ where: {
+ id: folderId,
+ },
+ select: {
+ projectId: true,
+ },
+ });
+
+ if (!folder) {
+ console.error(
+ `[folderDeletedJob] Folder ${folderId} not found. Skipping...`,
+ );
+ return;
+ }
+
+ if (folder.projectId !== "") {
+ console.error(
+ `[folderDeletedJob] Folder ${folderId} not marked for deletion. Skipping...`,
+ );
+ return;
+ }
+
+ const linksToUpdate = await prisma.link.findMany({
+ where: {
+ folderId,
+ },
+ take: MAX_LINKS_PER_BATCH,
+ orderBy: {
+ createdAt: "desc", // TODO we need to add [folderId, createdAt] index on Link table
+ },
+ include: {
+ ...includeTags,
+ ...includeProgramEnrollment,
+ },
+ });
+
+ if (linksToUpdate.length > 0) {
+ await recordLink(
+ linksToUpdate.map((link) => ({
+ ...link,
+ folderId: null,
+ })),
+ );
+
+ const { count } = await prisma.link.updateMany({
+ where: {
+ id: {
+ in: linksToUpdate.map((link) => link.id),
+ },
+ },
+ data: {
+ folderId: null,
+ },
+ });
+
+ console.log(
+ `[folderDeletedJob] Updated ${count} links in folder ${folderId}.`,
+ );
+ }
+
+ if (linksToUpdate.length === MAX_LINKS_PER_BATCH) {
+ await folderDeletedJob.dispatch(
+ {
+ folderId,
+ },
+ {
+ delay: 1,
+ label: folderId,
+ },
+ );
+
+ return;
+ }
+
+ await prisma.folder.delete({
+ where: {
+ id: folderId,
+ },
+ });
+
+ console.log(`[folderDeletedJob] Folder ${folderId} deleted from database.`);
+ },
+});
diff --git a/apps/web/lib/jobs/handlers/partner-tag-deleted-job.ts b/apps/web/lib/jobs/handlers/partner-tag-deleted-job.ts
new file mode 100644
index 0000000000..2f86106036
--- /dev/null
+++ b/apps/web/lib/jobs/handlers/partner-tag-deleted-job.ts
@@ -0,0 +1,118 @@
+import * as z from "zod/v4";
+import { includeProgramEnrollment } from "../../api/links/include-program-enrollment";
+import { includeTags } from "../../api/links/include-tags";
+import { CRON_BATCH_SIZE } from "../../cron";
+import { prisma } from "../../prisma";
+import { recordLink } from "../../tinybird";
+import { defineJob } from "../index";
+
+const inputSchema = z.object({
+ partnerTagId: z.string(),
+});
+
+// Job to delete a partner tag and all its program–partner tag associations
+export const partnerTagDeletedJob = defineJob({
+ name: "partner-tag-deleted-job",
+ schema: inputSchema,
+ async handle(input) {
+ const { partnerTagId } = input;
+
+ const partnerTag = await prisma.partnerTag.findUnique({
+ where: {
+ id: partnerTagId,
+ },
+ select: {
+ programId: true,
+ },
+ });
+
+ if (!partnerTag) {
+ console.error(
+ `[partnerTagDeletedJob] Partner tag ${partnerTagId} not found. Skipping...`,
+ );
+ return;
+ }
+
+ if (partnerTag.programId !== null) {
+ console.error(
+ `[partnerTagDeletedJob] Partner tag ${partnerTagId} not marked for deletion. Skipping...`,
+ );
+ return;
+ }
+
+ const programPartnerTags = await prisma.programPartnerTag.findMany({
+ where: {
+ partnerTagId,
+ },
+ select: {
+ id: true,
+ programEnrollment: {
+ select: {
+ id: true,
+ },
+ },
+ },
+ take: CRON_BATCH_SIZE,
+ });
+
+ if (programPartnerTags.length > 0) {
+ // Delete the program–partner tag associations.
+ const { count } = await prisma.programPartnerTag.deleteMany({
+ where: {
+ id: {
+ in: programPartnerTags.map(({ id }) => id),
+ },
+ },
+ });
+
+ console.log(
+ `[partnerTagDeletedJob] Deleted ${count} program–partner tag associations.`,
+ );
+
+ // Update the links to remove the partner tag from TB
+ const linksToUpdate = await prisma.link.findMany({
+ where: {
+ programEnrollment: {
+ id: {
+ in: programPartnerTags.map((row) => row.programEnrollment.id),
+ },
+ },
+ },
+ include: {
+ ...includeTags,
+ ...includeProgramEnrollment,
+ },
+ });
+
+ if (linksToUpdate.length > 0) {
+ await recordLink(linksToUpdate);
+ }
+
+ // More associations remain — queue the next batch.
+ if (programPartnerTags.length === CRON_BATCH_SIZE) {
+ await partnerTagDeletedJob.dispatch(
+ {
+ partnerTagId,
+ },
+ {
+ delay: 1,
+ label: partnerTagId,
+ },
+ );
+
+ return;
+ }
+ }
+
+ // No more program–partner tag associations left. Delete the partner tag.
+ await prisma.partnerTag.delete({
+ where: {
+ id: partnerTagId,
+ },
+ });
+
+ console.log(
+ `[partnerTagDeletedJob] Partner tag ${partnerTagId} deleted from database.`,
+ );
+ },
+});
diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.ts
new file mode 100644
index 0000000000..4e1afd59d5
--- /dev/null
+++ b/apps/web/lib/jobs/index.ts
@@ -0,0 +1,448 @@
+import { createId } from "@/lib/api/create-id";
+import { logger } from "@/lib/axiom/server";
+import { qstash } from "@/lib/cron";
+import { prisma } from "@/lib/prisma";
+import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";
+import { Prisma } from "@prisma/client";
+import { PublishRequest } from "@upstash/qstash";
+import * as z from "zod/v4";
+
+export type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;
+
+type JobPublishOptions = Pick<
+ PublishRequest,
+ | "delay"
+ | "notBefore"
+ | "deduplicationId"
+ | "retries"
+ | "flowControl"
+ | "label"
+>;
+
+export type JobDispatchOptions = JobPublishOptions & {
+ queue?: string;
+};
+
+// Per-job defaults, merged under per-dispatch options
+export type JobDefaults = Pick<
+ JobDispatchOptions,
+ "retries" | "queue" | "flowControl" | "label"
+>;
+
+export type DispatchResult =
+ | { status: "published"; messageId: string }
+ | { status: "deferred"; backgroundJobId: string };
+
+export type DispatchBatchResult = {
+ published: number;
+ deferred: number;
+ failed: number;
+ results: DispatchResult[];
+};
+
+interface DispatchJobInput {
+ name: string;
+ payload: unknown;
+ options?: JobDispatchOptions;
+}
+
+type JobReplayOptions = Pick<
+ JobDispatchOptions,
+ "deduplicationId" | "retries" | "queue" | "flowControl" | "label"
+>;
+
+const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;
+
+const QSTASH_PUBLISH_MAX_RETRIES = 3;
+
+const QSTASH_BATCH_CHUNK_SIZE = 100;
+
+export const jobNameSchema = z
+ .string()
+ .regex(
+ /^[a-z][a-z0-9]*(-[a-z0-9]+)*-job$/,
+ 'Job name must be kebab-case ending in "-job"',
+ );
+
+export function getJobsEndpointUrl(name: string) {
+ return `${JOBS_ENDPOINT_URL}/${name}`;
+}
+
+// QStash label: user-provided tag first, job name appended for log filtering
+export function buildJobLabel(name: string, label?: string) {
+ return label ? `${label},${name}` : name;
+}
+
+// QStash deduplicationId: user-provided id first, job name appended for cross-job isolation
+export function buildJobDeduplicationId(
+ name: string,
+ deduplicationId?: string,
+) {
+ if (!deduplicationId) {
+ return undefined;
+ }
+
+ return `${deduplicationId},${name}`;
+}
+
+// Wire format published to QStash and consumed by /api/jobs/process/[jobName]
+export const jobEnvelopeSchema = z.object({
+ name: jobNameSchema,
+ dispatchedAt: z.string(),
+ payload: z.unknown(),
+});
+
+function toErrorFields(error: unknown) {
+ return {
+ errorName: error instanceof Error ? error.name : undefined,
+ errorMessage: error instanceof Error ? error.message : String(error),
+ };
+}
+
+async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {
+ for (let attempt = 0; attempt <= QSTASH_PUBLISH_MAX_RETRIES; attempt++) {
+ try {
+ return await fn();
+ } catch (error) {
+ if (attempt < QSTASH_PUBLISH_MAX_RETRIES) {
+ await new Promise((resolve) =>
+ setTimeout(resolve, 1000 * Math.pow(2, attempt)),
+ );
+ continue;
+ }
+
+ throw error;
+ }
+ }
+
+ throw new Error("Failed to publish to QStash.");
+}
+
+function buildQStashJobRequest(
+ { name, payload, options }: DispatchJobInput,
+ opts?: {
+ dispatchedAt?: string;
+ batch?: boolean;
+ notBefore?: number;
+ },
+) {
+ const envelope: JobEnvelope = {
+ name,
+ payload,
+ dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),
+ };
+
+ const notBefore = opts?.notBefore ?? options?.notBefore;
+ const deduplicationId = buildJobDeduplicationId(
+ name,
+ options?.deduplicationId,
+ );
+
+ return {
+ url: getJobsEndpointUrl(name),
+ body: envelope,
+ label: buildJobLabel(name, options?.label),
+ ...(options?.delay &&
+ opts?.notBefore === undefined && {
+ delay: options.delay,
+ }),
+ ...(notBefore && { notBefore }),
+ ...(deduplicationId && { deduplicationId }),
+ ...(options?.retries !== undefined && { retries: options.retries }),
+ ...(options?.flowControl && { flowControl: options.flowControl }),
+ ...(opts?.batch && options?.queue && { queueName: options.queue }),
+ };
+}
+
+export function buildReplayRequest(
+ job: {
+ name: string;
+ payload: Prisma.JsonValue;
+ options: Prisma.JsonValue | null;
+ createdAt: Date;
+ scheduledFor: Date | null;
+ },
+ now: Date = new Date(),
+) {
+ const options = (job.options ?? {}) as JobReplayOptions;
+
+ const notBefore =
+ job.scheduledFor && job.scheduledFor > now
+ ? Math.floor(job.scheduledFor.getTime() / 1000)
+ : undefined;
+
+ return buildQStashJobRequest(
+ {
+ name: job.name,
+ payload: job.payload,
+ options,
+ },
+ {
+ dispatchedAt: job.createdAt.toISOString(),
+ batch: true,
+ notBefore,
+ },
+ );
+}
+
+export function isPublishSuccess(
+ response: unknown,
+): response is { messageId: string } {
+ return (
+ typeof response === "object" &&
+ response !== null &&
+ "messageId" in response &&
+ typeof response.messageId === "string"
+ );
+}
+
+// Persist jobs that could not be published to QStash. The
+// /api/cron/queue/retry cron republishes them and deletes the rows on success.
+async function persistBackgroundJobs(inputs: DispatchJobInput[]) {
+ const jobs = inputs.map(({ name, payload, options }) => {
+ let scheduledFor: Date | null = null;
+
+ if (options?.notBefore) {
+ scheduledFor = new Date(options.notBefore * 1000);
+ } else if (typeof options?.delay === "number") {
+ scheduledFor = new Date(Date.now() + options.delay * 1000);
+ }
+
+ const replayOptions = {
+ ...(options?.deduplicationId && {
+ deduplicationId: options.deduplicationId,
+ }),
+ ...(options?.retries !== undefined && { retries: options.retries }),
+ ...(options?.queue && { queue: options.queue }),
+ ...(options?.flowControl && { flowControl: options.flowControl }),
+ ...(options?.label && { label: options.label }),
+ };
+
+ return {
+ id: createId({ prefix: "job_" }),
+ name,
+ payload: payload as Prisma.InputJsonValue,
+ options: replayOptions as Prisma.InputJsonValue,
+ scheduledFor,
+ };
+ });
+
+ await prisma.job.createMany({
+ data: jobs,
+ });
+
+ return jobs;
+}
+
+async function deferJobs(inputs: DispatchJobInput[]) {
+ const backgroundJobs = await persistBackgroundJobs(inputs);
+
+ return backgroundJobs.map((backgroundJob) => ({
+ status: "deferred" as const,
+ backgroundJobId: backgroundJob.id,
+ }));
+}
+
+async function publishJobsToQStash(inputs: DispatchJobInput[]) {
+ if (inputs.length === 0) {
+ return [];
+ }
+
+ if (inputs.length === 1) {
+ const input = inputs[0];
+ const request = buildQStashJobRequest(input);
+
+ const response = await withQStashRetry(async () => {
+ if (input.options?.queue) {
+ return qstash
+ .queue({ queueName: input.options.queue })
+ .enqueueJSON(request);
+ }
+
+ return qstash.publishJSON(request);
+ });
+
+ return [response];
+ }
+
+ const requests = inputs.map((input) =>
+ buildQStashJobRequest(input, { batch: true }),
+ );
+
+ return withQStashRetry(() => qstash.batchJSON(requests));
+}
+
+async function dispatchJobs(
+ inputs: DispatchJobInput[],
+): Promise<DispatchBatchResult> {
+ if (inputs.length === 0) {
+ return {
+ published: 0,
+ deferred: 0,
+ failed: 0,
+ results: [],
+ };
+ }
+
+ const results: DispatchResult[] = [];
+ let published = 0;
+ let deferred = 0;
+ let failed = 0;
+ const isSingleDispatch = inputs.length === 1;
+
+ for (const inputChunk of chunk(inputs, QSTASH_BATCH_CHUNK_SIZE)) {
+ const jobName = inputChunk[0].name;
+
+ try {
+ const responses = await publishJobsToQStash(inputChunk);
+ const failedInputs: DispatchJobInput[] = [];
+
+ for (let index = 0; index < inputChunk.length; index++) {
+ const input = inputChunk[index];
+ const response = responses[index];
+
+ if (isPublishSuccess(response)) {
+ published++;
+ results.push({
+ status: "published",
+ messageId: response.messageId,
+ });
+
+ if (isSingleDispatch) {
+ console.log(`[jobs:${input.name}] published`, {
+ jobName: input.name,
+ label: buildJobLabel(input.name, input.options?.label),
+ messageId: response.messageId,
+ queue: input.options?.queue,
+ delay: input.options?.delay,
+ notBefore: input.options?.notBefore,
+ });
+ }
+
+ continue;
+ }
+
+ failedInputs.push(input);
+ }
+
+ if (failedInputs.length > 0) {
+ logger.error("jobs.publish_failed", {
+ jobName,
+ jobCount: failedInputs.length,
+ batch: !isSingleDispatch,
+ });
+
+ try {
+ const deferredResults = await deferJobs(failedInputs);
+ deferred += deferredResults.length;
+ results.push(...deferredResults);
+ } catch (error) {
+ logger.error("jobs.dispatch_lost", {
+ jobName,
+ jobCount: failedInputs.length,
+ ...toErrorFields(error),
+ });
+
+ failed += failedInputs.length;
+ await logger.flush();
+ throw error;
+ }
+ }
+
+ if (!isSingleDispatch) {
+ console.log(`[jobs:${jobName}] batch published`, {
+ jobName,
+ chunkSize: inputChunk.length,
+ published: inputChunk.length - failedInputs.length,
+ deferred: failedInputs.length,
+ });
+ }
+ } catch (error) {
+ logger.error("jobs.publish_failed", {
+ jobName,
+ jobCount: inputChunk.length,
+ batch: !isSingleDispatch,
+ ...toErrorFields(error),
+ });
+
+ try {
+ const deferredResults = await deferJobs(inputChunk);
+ deferred += deferredResults.length;
+ results.push(...deferredResults);
+ } catch (persistError) {
+ logger.error("jobs.dispatch_lost", {
+ jobName,
+ jobCount: inputChunk.length,
+ ...toErrorFields(persistError),
+ });
+
+ failed += inputChunk.length;
+ await logger.flush();
+ throw persistError;
+ }
+ }
+ }
+
+ await logger.flush();
+
+ return {
+ published,
+ deferred,
+ failed,
+ results,
+ };
+}
+
+export function defineJob<TSchema extends z.ZodType>({
+ name,
+ schema,
+ defaults,
+ handle,
+}: {
+ name: string;
+ schema: TSchema;
+ defaults?: JobDefaults;
+ handle: (payload: z.infer<TSchema>) => Promise<void>;
+}) {
+ jobNameSchema.parse(name);
+
+ return {
+ name,
+
+ execute: async (payload: unknown) => {
+ const parsed = schema.parse(payload);
+ await handle(parsed);
+ },
+
+ dispatch: async (
+ payload: z.infer<TSchema>,
+ options?: JobDispatchOptions,
+ ) => {
+ const { results } = await dispatchJobs([
+ {
+ name,
+ payload,
+ options: { ...defaults, ...options },
+ },
+ ]);
+ return results[0];
+ },
+
+ dispatchBatch: (
+ payloads: z.infer<TSchema>[],
+ getOptions?: (
+ payload: z.infer<TSchema>,
+ index: number,
+ ) => JobDispatchOptions | undefined,
+ ) =>
+ dispatchJobs(
+ payloads.map((payload, index) => ({
+ name,
+ payload,
+ options: { ...defaults, ...getOptions?.(payload, index) },
+ })),
+ ),
+ };
+}
+
+export type JobDefinition = ReturnType<typeof defineJob<z.ZodType>>;
diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.ts
new file mode 100644
index 0000000000..85fd5ef652
--- /dev/null
+++ b/apps/web/lib/jobs/registry.ts
@@ -0,0 +1,35 @@
+import type { JobDefinition } from "./index";
+
+// Add handlers/{name}-job.ts. Set defineJob({ name: "{name}-job" }).
+// Each entry uses a static import() so webpack code-splits handlers into separate chunks.
+const jobLoaders = {
+ "folder-deleted-job": () =>
+ import("./handlers/folder-deleted-job").then((m) => m.folderDeletedJob),
+
+ "partner-tag-deleted-job": () =>
+ import("./handlers/partner-tag-deleted-job").then(
+ (m) => m.partnerTagDeletedJob,
+ ),
+} as const satisfies Record<string, () => Promise<JobDefinition>>;
+
+const jobCache = new Map<string, JobDefinition>();
+
+export async function loadJob(
+ name: string,
+): Promise<JobDefinition | undefined> {
+ const cached = jobCache.get(name);
+ if (cached) return cached;
+
+ const loader = jobLoaders[name as keyof typeof jobLoaders];
+ if (!loader) return undefined;
+
+ const job = await loader();
+ if (job.name !== name) {
+ throw new Error(`Job name mismatch: ${job.name} !== ${name}`);
+ }
+
+ jobCache.set(name, job);
+ return job;
+}
+
+export const registeredJobNames = Object.keys(jobLoaders);
diff --git a/apps/web/prisma/schema/job.prisma b/apps/web/prisma/schema/job.prisma
new file mode 100644
index 0000000000..dbc55bae01
--- /dev/null
+++ b/apps/web/prisma/schema/job.prisma
@@ -0,0 +1,15 @@
+// Background jobs that could not be published to QStash (e.g. QStash outage).
+// The /api/cron/queue/retry cron republishes these and deletes them on success.
+model Job {
+ id String @id
+ name String
+ payload Json
+ options Json? // dispatch options replayed verbatim: deduplicationId, retries, queue, flowControl, label
+ scheduledFor DateTime? // absolute time the job should run (derived from delay/notBefore at dispatch time)
+ attempts Int @default(0)
+ lastError String? @db.Text
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+
+ @@index([createdAt])
+}
diff --git a/apps/web/vercel.json b/apps/web/vercel.json
index ab9891b717..91c88525c6 100644
--- a/apps/web/vercel.json
+++ b/apps/web/vercel.json
@@ -83,6 +83,10 @@
{
"path": "/api/cron/webhooks/sync-click-workspaces",
"schedule": "*/5 * * * *"
+ },
+ {
+ "path": "/api/cron/queue/retry",
+ "schedule": "* * * * *"
}
],
"functions": {