The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Add durable background job framework with DB-backed QStash retry (#4130)
Kiran K96f6a39102bc
diff --git a/apps/web/app/(ee)/api/cron/folders/delete/route.ts b/apps/web/app/(ee)/api/cron/folders/delete/route.tsdeleted file mode 100644index 346a75500e..0000000000--- a/apps/web/app/(ee)/api/cron/folders/delete/route.ts+++ /dev/null@@ -1,95 +0,0 @@-import { handleAndReturnErrorResponse } from "@/lib/api/errors";-import { queueFolderDeletion } from "@/lib/api/folders/queue-folder-deletion";-import { includeProgramEnrollment } from "@/lib/api/links/include-program-enrollment";-import { includeTags } from "@/lib/api/links/include-tags";-import { verifyQstashSignature } from "@/lib/cron/verify-qstash";-import { prisma } from "@/lib/prisma";-import { recordLink } from "@/lib/tinybird";-import * as z from "zod/v4";--export const dynamic = "force-dynamic";--const MAX_LINKS_PER_BATCH = 500;--const schema = z.object({- folderId: z.string(),-});--// POST /api/cron/folders/delete-// Recursively remove the `folderId` association in all the links of a folder from Tinybird-export async function POST(req: Request) {- try {- const rawBody = await req.text();- await verifyQstashSignature({ req, rawBody });-- const { folderId } = schema.parse(JSON.parse(rawBody));-- const linksToUpdate = await prisma.link.findMany({- where: {- folderId,- },- take: MAX_LINKS_PER_BATCH,- orderBy: {- createdAt: "desc", // TODO we need to add [folderId, createdAt] index on Link table- },- include: {- ...includeTags,- ...includeProgramEnrollment,- },- });-- if (linksToUpdate.length === 0) {- await prisma.projectUsers.updateMany({- where: {- defaultFolderId: folderId,- },- data: {- defaultFolderId: null,- },- });-- await prisma.folder.delete({- where: {- id: folderId,- },- });-- return new Response("No more links to process. Deleting folder...");- }-- const recordLinkResponse = await recordLink(- linksToUpdate.map((link) => ({- ...link,- folderId: null,- })),- );-- console.log("recordLinkResponse", recordLinkResponse);-- const updateLinksResponse = await prisma.link.updateMany({- where: {- id: {- in: linksToUpdate.map((link) => link.id),- },- },- data: {- folderId: null,- },- });-- console.log("updateLinksResponse", updateLinksResponse);-- // TODO: technically we can check if linksToUpdate.length < MAX_LINKS_PER_BATCH- // because that means all the links have been updated and we can delete the folder- await queueFolderDeletion({- folderId,- delay: 2,- });-- return new Response(- `Processed ${linksToUpdate.length} links in the folder.`,- );- } catch (error) {- return handleAndReturnErrorResponse(error);- }-}diff --git a/apps/web/app/(ee)/api/cron/queue/retry/route.ts b/apps/web/app/(ee)/api/cron/queue/retry/route.tsnew file mode 100644index 0000000000..1c1af54078--- /dev/null+++ b/apps/web/app/(ee)/api/cron/queue/retry/route.ts@@ -0,0 +1,168 @@+import { logger } from "@/lib/axiom/server";+import { qstash } from "@/lib/cron";+import { withCron } from "@/lib/cron/with-cron";+import { buildReplayRequest, isPublishSuccess } from "@/lib/jobs";+import { prisma } from "@/lib/prisma";+import { redis } from "@/lib/upstash/redis";+import { logAndRespond } from "../../utils";++export const dynamic = "force-dynamic";++const MAX_ATTEMPTS = 10;+const BATCH_SIZE = 100;++// Lock for the cron job. TTL must be ≥ cron maxDuration (600s in vercel.json)+// so the lock cannot expire while a run is still alive and allow a concurrent+// minute-cron invocation+const LOCK_KEY = "lock:queue-retry";+const LOCK_TTL_SECONDS = 600;++// GET /api/cron/queue/retry – republish background jobs that failed to+// publish to QStash at dispatch time; rows are deleted on successful publish+export const GET = withCron(async () => {+ const acquired = await redis.set(LOCK_KEY, "1", {+ nx: true,+ ex: LOCK_TTL_SECONDS,+ });++ if (!acquired) {+ return logAndRespond(+ "[queue-retry] Another run is in progress. Skipping...",+ );+ }++ try {+ const jobs = await prisma.job.findMany({+ where: {+ attempts: {+ lt: MAX_ATTEMPTS,+ },+ },+ orderBy: {+ createdAt: "asc",+ },+ take: BATCH_SIZE,+ });++ if (jobs.length === 0) {+ return logAndRespond("No background jobs to retry.");+ }++ const now = new Date();+ const entries = jobs.map((job) => buildReplayRequest(job, now));+ const jobIds = jobs.map(({ id }) => id);+ let responses: unknown[];++ try {+ responses = await qstash.batchJSON(entries);+ } catch (error) {+ const errorMessage =+ error instanceof Error ? error.message : String(error);++ await prisma.job.updateMany({+ where: {+ id: {+ in: jobIds,+ },+ },+ data: {+ attempts: {+ increment: 1,+ },+ lastError: errorMessage.slice(0, 1000),+ },+ });++ logger.error("jobs.retry_failed", {+ jobCount: jobs.length,+ errorMessage,+ });++ // Jobs that just ran out of attempts are excluded from future runs by the+ // MAX_ATTEMPTS filter and need manual intervention+ const exhaustedJobs = jobs.filter(+ (backgroundJob) => backgroundJob.attempts + 1 >= MAX_ATTEMPTS,+ );++ if (exhaustedJobs.length > 0) {+ logger.error("jobs.retry_exhausted", {+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),+ });+ }++ await logger.flush();++ return logAndRespond(+ `Failed to republish ${jobs.length} background jobs to QStash.`,+ );+ }++ const publishedJobIds: string[] = [];+ const failedJobs: typeof jobs = [];++ jobs.forEach((job, index) => {+ if (isPublishSuccess(responses[index])) {+ publishedJobIds.push(job.id);+ } else {+ failedJobs.push(job);+ }+ });++ if (publishedJobIds.length > 0) {+ await prisma.job.deleteMany({+ where: {+ id: {+ in: publishedJobIds,+ },+ },+ });+ }++ if (failedJobs.length === 0) {+ return logAndRespond(+ `Republished ${jobs.length} background jobs to QStash.`,+ );+ }++ const errorMessage = "QStash batch publish did not return a messageId";++ await prisma.job.updateMany({+ where: {+ id: {+ in: failedJobs.map(({ id }) => id),+ },+ },+ data: {+ attempts: {+ increment: 1,+ },+ lastError: errorMessage,+ },+ });++ logger.error("jobs.retry_failed", {+ jobCount: failedJobs.length,+ errorMessage,+ });++ // Jobs that just ran out of attempts are excluded from future runs by the+ // MAX_ATTEMPTS filter and need manual intervention+ const exhaustedJobs = failedJobs.filter(+ (backgroundJob) => backgroundJob.attempts + 1 >= MAX_ATTEMPTS,+ );++ if (exhaustedJobs.length > 0) {+ logger.error("jobs.retry_exhausted", {+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),+ });+ }++ await logger.flush();++ return logAndRespond(+ `Republished ${publishedJobIds.length} background jobs to QStash; failed to republish ${failedJobs.length}.`,+ );+ } finally {+ await redis.del(LOCK_KEY);+ }+});diff --git a/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts b/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.tsindex d119666526..c7edc277d1 100644--- a/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts+++ b/apps/web/app/(ee)/api/cron/streams/send-link-clicked-webhooks/route.ts@@ -20,9 +20,11 @@ const BATCH_SIZE = 6000;// How many webhook deliveries to enqueue to QStash in parallelconst SEND_CONCURRENCY = 100;-// Lock for the cron job+// Lock for the cron job. TTL must be ≥ cron maxDuration (600s in vercel.json)+// so the lock cannot expire while a run is still alive and allow a concurrent+// minute-cron invocationconst LOCK_KEY = "lock:send-link-clicked-webhooks";-const LOCK_TTL_SECONDS = 120;+const LOCK_TTL_SECONDS = 600;// Drains the workspace:click:events stream and delivers link.clicked webhooks.// Runs every minute (see vercel.json).diff --git a/apps/web/app/(ee)/api/cron/tags/delete-partner-tag/route.ts b/apps/web/app/(ee)/api/cron/tags/delete-partner-tag/route.tsdeleted file mode 100644index 436d2680a1..0000000000--- a/apps/web/app/(ee)/api/cron/tags/delete-partner-tag/route.ts+++ /dev/null@@ -1,100 +0,0 @@-import { handleAndReturnErrorResponse } from "@/lib/api/errors";-import { includeProgramEnrollment } from "@/lib/api/links/include-program-enrollment";-import { includeTags } from "@/lib/api/links/include-tags";-import { queuePartnerTagDeletion } from "@/lib/api/tags/queue-tag-deletion";-import { CRON_BATCH_SIZE } from "@/lib/cron";-import { verifyQstashSignature } from "@/lib/cron/verify-qstash";-import { prisma } from "@/lib/prisma";-import { recordLink } from "@/lib/tinybird";-import * as z from "zod/v4";-import { logAndRespond } from "../../utils";--export const dynamic = "force-dynamic";--const schema = z.object({- id: z.string(),-});--// POST /api/cron/tags/delete-partner-tag-export async function POST(req: Request) {- try {- const rawBody = await req.text();- await verifyQstashSignature({ req, rawBody });-- const { id: partnerTagId } = schema.parse(JSON.parse(rawBody));-- const partnerTag = await prisma.partnerTag.findUnique({- where: {- id: partnerTagId,- },- });-- if (!partnerTag || partnerTag.programId !== null) {- return logAndRespond(- `Partner tag ${partnerTagId} not found or not marked for deletion. Skipping...`,- );- }-- const programPartnerTags = await prisma.programPartnerTag.findMany({- where: {- partnerTagId,- },- select: {- id: true,- programEnrollment: {- select: {- id: true,- },- },- },- take: CRON_BATCH_SIZE,- });-- if (programPartnerTags.length === 0) {- await prisma.partnerTag.delete({- where: {- id: partnerTagId,- },- });-- return logAndRespond(- `No ProgramPartnerTag rows left. Partner tag ${partnerTagId} deleted from database.`,- );- }-- const { count } = await prisma.programPartnerTag.deleteMany({- where: {- id: {- in: programPartnerTags.map((row) => row.id),- },- },- });- console.log(`Deleted ${count} program–partner tag associations`);-- const linksToUpdate = await prisma.link.findMany({- where: {- programEnrollment: {- id: {- in: programPartnerTags.map((row) => row.programEnrollment.id),- },- },- },- include: {- ...includeTags,- ...includeProgramEnrollment,- },- });-- console.log(`Found ${linksToUpdate.length} links to update in Tinybird`);-- await recordLink(linksToUpdate);-- await queuePartnerTagDeletion({- partnerTagId,- delay: 2,- });- return logAndRespond("Batch completed, starting next batch...");- } catch (error) {- return handleAndReturnErrorResponse(error);- }-}diff --git a/apps/web/app/api/folders/[folderId]/route.ts b/apps/web/app/api/folders/[folderId]/route.tsindex ff9913723a..270bcb41ea 100644--- a/apps/web/app/api/folders/[folderId]/route.ts+++ b/apps/web/app/api/folders/[folderId]/route.ts@@ -1,8 +1,8 @@import { DubApiError } from "@/lib/api/errors";-import { queueFolderDeletion } from "@/lib/api/folders/queue-folder-deletion";import { parseRequestBody } from "@/lib/api/utils";import { withWorkspace } from "@/lib/auth";import { verifyFolderAccess } from "@/lib/folder/permissions";+import { folderDeletedJob } from "@/lib/jobs/handlers/folder-deleted-job";import { getPlanCapabilities } from "@/lib/plan-capabilities";import { prisma } from "@/lib/prisma";import { FolderSchema, updateFolderSchema } from "@/lib/zod/schemas/folders";@@ -127,43 +127,41 @@ export const DELETE = withWorkspace(where: {folderId,},+ take: 100,});- // if there are no links associated with the folder, we can just delete it- if (linksCount === 0) {- await prisma.folder.delete({+ await prisma.$transaction([+ // If there are no links associated with the folder, we can just delete it+ ...(linksCount === 0+ ? [+ prisma.folder.delete({+ where: {+ id: folderId,+ },+ }),+ ]+ : [+ prisma.folder.update({+ where: {+ id: folderId,+ },+ data: {+ projectId: "",+ },+ }),+ ]),++ // Remove the default folder assignment for all users whose defaultFolderId matches the given folderId+ prisma.projectUsers.updateMany({where: {- id: folderId,+ defaultFolderId: folderId,},- });- } else {- await Promise.all([- prisma.folder.update({- where: {- id: folderId,- },- data: {- projectId: "",- },- }),-- queueFolderDeletion({- folderId,- }),- ]);- }-- // Remove the default folder assignment for all users whose defaultFolderId matches the given folderId- await prisma.projectUsers.updateMany({- where: {- defaultFolderId: folderId,- },- data: {- defaultFolderId: null,- },- });+ data: {+ defaultFolderId: null,+ },+ }),- waitUntil(+ // Decrement the folders usage for the workspaceprisma.project.update({where: {id: workspace.id,@@ -174,7 +172,20 @@ export const DELETE = withWorkspace(},},}),- );+ ]);++ if (linksCount > 0) {+ waitUntil(+ folderDeletedJob.dispatch(+ {+ folderId,+ },+ {+ label: folderId,+ },+ ),+ );+ }return NextResponse.json({ id: folderId });},diff --git a/apps/web/app/api/jobs/process/[jobName]/route.ts b/apps/web/app/api/jobs/process/[jobName]/route.tsnew file mode 100644index 0000000000..aad8b1f2d9--- /dev/null+++ b/apps/web/app/api/jobs/process/[jobName]/route.ts@@ -0,0 +1,101 @@+import { logger, withAxiomBodyLog } from "@/lib/axiom/server";+import { verifyQstashSignature } from "@/lib/cron/verify-qstash";+import { jobEnvelopeSchema } from "@/lib/jobs";+import { loadJob } from "@/lib/jobs/registry";+import * as z from "zod/v4";++export const maxDuration = 600;++// POST /api/jobs/process/[jobName] – shared executor for all background jobs+export const POST = withAxiomBodyLog(+ async (+ req: Request,+ { params }: { params: Promise<{ jobName: string }> },+ ) => {+ const { jobName: urlJobName } = await params;+ const clonedReq = req.clone();+ const rawBody = await clonedReq.text();++ await verifyQstashSignature({ req, rawBody });++ let parsedBody: unknown = null;++ try {+ parsedBody = JSON.parse(rawBody);+ } catch (error) {+ // Handled below by the envelope validation+ }++ const envelope = jobEnvelopeSchema.safeParse(parsedBody);++ // Permanently invalid — returning 2xx stops QStash retries+ if (!envelope.success) {+ logger.error("jobs.invalid_envelope", {+ error: envelope.error.message,+ });+ await logger.flush();++ return new Response("Invalid job envelope (non-retryable).");+ }++ const { name: jobName } = envelope.data;++ if (urlJobName !== jobName) {+ logger.error("jobs.job_name_mismatch", {+ urlJobName,+ envelopeJobName: jobName,+ });+ await logger.flush();++ return new Response(+ `Job name mismatch: URL "${urlJobName}" does not match envelope "${jobName}" (non-retryable).`,+ );+ }++ console.log(`[jobs:${jobName}] Executing job...`, envelope.data);++ // Get the job definition+ const job = await loadJob(jobName);++ if (!job) {+ logger.error("jobs.unknown_job", {+ jobName,+ });+ await logger.flush();++ return new Response(`Unknown job "${jobName}" (non-retryable).`);+ }++ try {+ await job.execute(envelope.data.payload);+ } catch (error) {+ if (error instanceof z.ZodError) {+ // A bad payload is permanent — 200 so QStash doesn't retry+ logger.error("jobs.invalid_payload", {+ jobName: job.name,+ error: error.message,+ });+ await logger.flush();++ return new Response(+ `Invalid payload for job "${job.name}" (non-retryable).`,+ );+ }++ // Transient failure → 500 → QStash retries+ logger.error("jobs.execution_failed", {+ jobName: job.name,+ errorName: error instanceof Error ? error.name : undefined,+ errorMessage: error instanceof Error ? error.message : String(error),+ errorStack: error instanceof Error ? error.stack : undefined,+ });+ await logger.flush();++ return new Response(`Job "${job.name}" failed.`, {+ status: 500,+ });+ }++ return new Response(`Executed job "${job.name}".`);+ },+);diff --git a/apps/web/lib/actions/partners/tags/delete-partner-tag.ts b/apps/web/lib/actions/partners/tags/delete-partner-tag.tsindex 51fb58fa66..adeeb93ee1 100644--- a/apps/web/lib/actions/partners/tags/delete-partner-tag.ts+++ b/apps/web/lib/actions/partners/tags/delete-partner-tag.ts@@ -1,8 +1,10 @@"use server";import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw";-import { markPartnerTagDeleted } from "@/lib/api/tags/mark-partner-tag-deleted";+import { partnerTagDeletedJob } from "@/lib/jobs/handlers/partner-tag-deleted-job";+import { prisma } from "@/lib/prisma";import { deletePartnerTagSchema } from "@/lib/zod/schemas/partner-tags";+import { waitUntil } from "@vercel/functions";import { authActionClient } from "../../safe-action";import { throwIfNoPermission } from "../../throw-if-no-permission";@@ -20,12 +22,28 @@ export const deletePartnerTagAction = authActionClientconst programId = getDefaultProgramIdOrThrow(workspace);- const deleted = await markPartnerTagDeleted({- partnerTagId,- programId,+ const { count } = await prisma.partnerTag.updateMany({+ where: {+ id: partnerTagId,+ programId,+ },+ data: {+ programId: null,+ },});- if (!deleted) {- throw new Error("Partner tag not found.");+ if (count === 0) {+ throw new Error("Partner tag not found or already deleted.");}++ waitUntil(+ partnerTagDeletedJob.dispatch(+ {+ partnerTagId,+ },+ {+ label: partnerTagId,+ },+ ),+ );});diff --git a/apps/web/lib/api/create-id.ts b/apps/web/lib/api/create-id.tsindex 3165325eaa..bb4a8b4814 100644--- a/apps/web/lib/api/create-id.ts+++ b/apps/web/lib/api/create-id.ts@@ -47,6 +47,7 @@ const prefixes = ["sbl_", // submitted lead"pb_", // partner postback"req_", // api request log+ "job_", // background job] as const;// ULID uses base32 encodingdiff --git a/apps/web/lib/api/folders/delete-workspace-folders.ts b/apps/web/lib/api/folders/delete-workspace-folders.tsindex f911c9fb80..38249edd3e 100644--- a/apps/web/lib/api/folders/delete-workspace-folders.ts+++ b/apps/web/lib/api/folders/delete-workspace-folders.ts@@ -1,5 +1,5 @@+import { folderDeletedJob } from "@/lib/jobs/handlers/folder-deleted-job";import { prisma } from "@/lib/prisma";-import { queueFolderDeletion } from "./queue-folder-deletion";export async function deleteWorkspaceFolders({workspaceId,@@ -20,6 +20,7 @@ export async function deleteWorkspaceFolders({});excludedFolderId = program.defaultFolderId;}+const folders = await prisma.folder.findMany({where: {projectId: workspaceId,@@ -39,22 +40,32 @@ export async function deleteWorkspaceFolders({return;}- await prisma.projectUsers.updateMany({- where: {- projectId: workspaceId,- defaultFolderId: { in: folders.map(({ id }) => id) },- },- data: {- defaultFolderId: null,- },- });+ const folderIds = folders.map(({ id }) => id);++ await prisma.$transaction([+ prisma.projectUsers.updateMany({+ where: {+ projectId: workspaceId,+ defaultFolderId: {+ in: folderIds,+ },+ },+ data: {+ defaultFolderId: null,+ },+ }),++ prisma.folder.updateMany({+ where: {+ id: {+ in: folderIds,+ },+ },+ data: {+ projectId: "",+ },+ }),- return await Promise.all([- ...folders.map(({ id }) =>- queueFolderDeletion({- folderId: id,- }),- ),prisma.project.update({where: {id: workspaceId,@@ -64,4 +75,11 @@ export async function deleteWorkspaceFolders({},}),]);++ await folderDeletedJob.dispatchBatch(+ folderIds.map((folderId) => ({ folderId })),+ ({ folderId }) => ({+ label: folderId,+ }),+ );}diff --git a/apps/web/lib/api/folders/queue-folder-deletion.ts b/apps/web/lib/api/folders/queue-folder-deletion.tsdeleted file mode 100644index 0bcaa4ff5b..0000000000--- a/apps/web/lib/api/folders/queue-folder-deletion.ts+++ /dev/null@@ -1,18 +0,0 @@-import { qstash } from "@/lib/cron";-import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";--export async function queueFolderDeletion({- folderId,- delay,-}: {- folderId: string;- delay?: number;-}) {- return await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/folders/delete`,- ...(delay && { delay }),- body: {- folderId,- },- });-}diff --git a/apps/web/lib/api/tags/mark-partner-tag-deleted.ts b/apps/web/lib/api/tags/mark-partner-tag-deleted.tsdeleted file mode 100644index ca067083d9..0000000000--- a/apps/web/lib/api/tags/mark-partner-tag-deleted.ts+++ /dev/null@@ -1,41 +0,0 @@-import { prisma } from "@/lib/prisma";-import { queuePartnerTagDeletion } from "./queue-tag-deletion";--// Mark the partner tag as deleted (orphan from program). ProgramPartnerTag-// rows and the PartnerTag row are removed asynchronously via a cron job.-export async function markPartnerTagDeleted({- partnerTagId,- programId,-}: {- partnerTagId: string;- programId: string;-}): Promise<boolean> {- try {- const updated = await prisma.partnerTag.update({- where: {- id: partnerTagId,- programId,- },- data: {- programId: null,- },- });-- if (!updated) {- return false;- }-- await queuePartnerTagDeletion({- partnerTagId,- });-- return true;- } catch (error) {- console.error("markPartnerTagDeleted", {- reason: error,- partnerTagId,- programId,- });- return false;- }-}diff --git a/apps/web/lib/api/tags/queue-tag-deletion.ts b/apps/web/lib/api/tags/queue-tag-deletion.tsindex 78a62db2bb..6e9af100df 100644--- a/apps/web/lib/api/tags/queue-tag-deletion.ts+++ b/apps/web/lib/api/tags/queue-tag-deletion.ts@@ -16,19 +16,3 @@ export async function queueLinkTagDeletion({},});}--export async function queuePartnerTagDeletion({- partnerTagId,- delay,-}: {- partnerTagId: string;- delay?: number;-}) {- return await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/tags/delete-partner-tag`,- ...(delay && { delay }),- body: {- id: partnerTagId,- },- });-}diff --git a/apps/web/lib/jobs/handlers/folder-deleted-job.ts b/apps/web/lib/jobs/handlers/folder-deleted-job.tsnew file mode 100644index 0000000000..83d4464b37--- /dev/null+++ b/apps/web/lib/jobs/handlers/folder-deleted-job.ts@@ -0,0 +1,104 @@+import * as z from "zod/v4";+import { includeProgramEnrollment } from "../../api/links/include-program-enrollment";+import { includeTags } from "../../api/links/include-tags";+import { prisma } from "../../prisma";+import { recordLink } from "../../tinybird";+import { defineJob } from "../index";++const MAX_LINKS_PER_BATCH = 500;++const inputSchema = z.object({+ folderId: z.string(),+});++// Job to remove folderId from all links and delete the folder+export const folderDeletedJob = defineJob({+ name: "folder-deleted-job",+ schema: inputSchema,+ async handle(input) {+ const { folderId } = input;++ const folder = await prisma.folder.findUnique({+ where: {+ id: folderId,+ },+ select: {+ projectId: true,+ },+ });++ if (!folder) {+ console.error(+ `[folderDeletedJob] Folder ${folderId} not found. Skipping...`,+ );+ return;+ }++ if (folder.projectId !== "") {+ console.error(+ `[folderDeletedJob] Folder ${folderId} not marked for deletion. Skipping...`,+ );+ return;+ }++ const linksToUpdate = await prisma.link.findMany({+ where: {+ folderId,+ },+ take: MAX_LINKS_PER_BATCH,+ orderBy: {+ createdAt: "desc", // TODO we need to add [folderId, createdAt] index on Link table+ },+ include: {+ ...includeTags,+ ...includeProgramEnrollment,+ },+ });++ if (linksToUpdate.length > 0) {+ await recordLink(+ linksToUpdate.map((link) => ({+ ...link,+ folderId: null,+ })),+ );++ const { count } = await prisma.link.updateMany({+ where: {+ id: {+ in: linksToUpdate.map((link) => link.id),+ },+ },+ data: {+ folderId: null,+ },+ });++ console.log(+ `[folderDeletedJob] Updated ${count} links in folder ${folderId}.`,+ );+ }++ if (linksToUpdate.length === MAX_LINKS_PER_BATCH) {+ await folderDeletedJob.dispatch(+ {+ folderId,+ },+ {+ delay: 1,+ label: folderId,+ },+ );++ return;+ }++ await prisma.folder.delete({+ where: {+ id: folderId,+ },+ });++ console.log(`[folderDeletedJob] Folder ${folderId} deleted from database.`);+ },+});diff --git a/apps/web/lib/jobs/handlers/partner-tag-deleted-job.ts b/apps/web/lib/jobs/handlers/partner-tag-deleted-job.tsnew file mode 100644index 0000000000..2f86106036--- /dev/null+++ b/apps/web/lib/jobs/handlers/partner-tag-deleted-job.ts@@ -0,0 +1,118 @@+import * as z from "zod/v4";+import { includeProgramEnrollment } from "../../api/links/include-program-enrollment";+import { includeTags } from "../../api/links/include-tags";+import { CRON_BATCH_SIZE } from "../../cron";+import { prisma } from "../../prisma";+import { recordLink } from "../../tinybird";+import { defineJob } from "../index";++const inputSchema = z.object({+ partnerTagId: z.string(),+});++// Job to delete a partner tag and all its program–partner tag associations+export const partnerTagDeletedJob = defineJob({+ name: "partner-tag-deleted-job",+ schema: inputSchema,+ async handle(input) {+ const { partnerTagId } = input;++ const partnerTag = await prisma.partnerTag.findUnique({+ where: {+ id: partnerTagId,+ },+ select: {+ programId: true,+ },+ });++ if (!partnerTag) {+ console.error(+ `[partnerTagDeletedJob] Partner tag ${partnerTagId} not found. Skipping...`,+ );+ return;+ }++ if (partnerTag.programId !== null) {+ console.error(+ `[partnerTagDeletedJob] Partner tag ${partnerTagId} not marked for deletion. Skipping...`,+ );+ return;+ }++ const programPartnerTags = await prisma.programPartnerTag.findMany({+ where: {+ partnerTagId,+ },+ select: {+ id: true,+ programEnrollment: {+ select: {+ id: true,+ },+ },+ },+ take: CRON_BATCH_SIZE,+ });++ if (programPartnerTags.length > 0) {+ // Delete the program–partner tag associations.+ const { count } = await prisma.programPartnerTag.deleteMany({+ where: {+ id: {+ in: programPartnerTags.map(({ id }) => id),+ },+ },+ });++ console.log(+ `[partnerTagDeletedJob] Deleted ${count} program–partner tag associations.`,+ );++ // Update the links to remove the partner tag from TB+ const linksToUpdate = await prisma.link.findMany({+ where: {+ programEnrollment: {+ id: {+ in: programPartnerTags.map((row) => row.programEnrollment.id),+ },+ },+ },+ include: {+ ...includeTags,+ ...includeProgramEnrollment,+ },+ });++ if (linksToUpdate.length > 0) {+ await recordLink(linksToUpdate);+ }++ // More associations remain — queue the next batch.+ if (programPartnerTags.length === CRON_BATCH_SIZE) {+ await partnerTagDeletedJob.dispatch(+ {+ partnerTagId,+ },+ {+ delay: 1,+ label: partnerTagId,+ },+ );++ return;+ }+ }++ // No more program–partner tag associations left. Delete the partner tag.+ await prisma.partnerTag.delete({+ where: {+ id: partnerTagId,+ },+ });++ console.log(+ `[partnerTagDeletedJob] Partner tag ${partnerTagId} deleted from database.`,+ );+ },+});diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.tsnew file mode 100644index 0000000000..4e1afd59d5--- /dev/null+++ b/apps/web/lib/jobs/index.ts@@ -0,0 +1,448 @@+import { createId } from "@/lib/api/create-id";+import { logger } from "@/lib/axiom/server";+import { qstash } from "@/lib/cron";+import { prisma } from "@/lib/prisma";+import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";+import { Prisma } from "@prisma/client";+import { PublishRequest } from "@upstash/qstash";+import * as z from "zod/v4";++export type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;++type JobPublishOptions = Pick<+ PublishRequest,+ | "delay"+ | "notBefore"+ | "deduplicationId"+ | "retries"+ | "flowControl"+ | "label"+>;++export type JobDispatchOptions = JobPublishOptions & {+ queue?: string;+};++// Per-job defaults, merged under per-dispatch options+export type JobDefaults = Pick<+ JobDispatchOptions,+ "retries" | "queue" | "flowControl" | "label"+>;++export type DispatchResult =+ | { status: "published"; messageId: string }+ | { status: "deferred"; backgroundJobId: string };++export type DispatchBatchResult = {+ published: number;+ deferred: number;+ failed: number;+ results: DispatchResult[];+};++interface DispatchJobInput {+ name: string;+ payload: unknown;+ options?: JobDispatchOptions;+}++type JobReplayOptions = Pick<+ JobDispatchOptions,+ "deduplicationId" | "retries" | "queue" | "flowControl" | "label"+>;++const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;++const QSTASH_PUBLISH_MAX_RETRIES = 3;++const QSTASH_BATCH_CHUNK_SIZE = 100;++export const jobNameSchema = z+ .string()+ .regex(+ /^[a-z][a-z0-9]*(-[a-z0-9]+)*-job$/,+ 'Job name must be kebab-case ending in "-job"',+ );++export function getJobsEndpointUrl(name: string) {+ return `${JOBS_ENDPOINT_URL}/${name}`;+}++// QStash label: user-provided tag first, job name appended for log filtering+export function buildJobLabel(name: string, label?: string) {+ return label ? `${label},${name}` : name;+}++// QStash deduplicationId: user-provided id first, job name appended for cross-job isolation+export function buildJobDeduplicationId(+ name: string,+ deduplicationId?: string,+) {+ if (!deduplicationId) {+ return undefined;+ }++ return `${deduplicationId},${name}`;+}++// Wire format published to QStash and consumed by /api/jobs/process/[jobName]+export const jobEnvelopeSchema = z.object({+ name: jobNameSchema,+ dispatchedAt: z.string(),+ payload: z.unknown(),+});++function toErrorFields(error: unknown) {+ return {+ errorName: error instanceof Error ? error.name : undefined,+ errorMessage: error instanceof Error ? error.message : String(error),+ };+}++async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {+ for (let attempt = 0; attempt <= QSTASH_PUBLISH_MAX_RETRIES; attempt++) {+ try {+ return await fn();+ } catch (error) {+ if (attempt < QSTASH_PUBLISH_MAX_RETRIES) {+ await new Promise((resolve) =>+ setTimeout(resolve, 1000 * Math.pow(2, attempt)),+ );+ continue;+ }++ throw error;+ }+ }++ throw new Error("Failed to publish to QStash.");+}++function buildQStashJobRequest(+ { name, payload, options }: DispatchJobInput,+ opts?: {+ dispatchedAt?: string;+ batch?: boolean;+ notBefore?: number;+ },+) {+ const envelope: JobEnvelope = {+ name,+ payload,+ dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),+ };++ const notBefore = opts?.notBefore ?? options?.notBefore;+ const deduplicationId = buildJobDeduplicationId(+ name,+ options?.deduplicationId,+ );++ return {+ url: getJobsEndpointUrl(name),+ body: envelope,+ label: buildJobLabel(name, options?.label),+ ...(options?.delay &&+ opts?.notBefore === undefined && {+ delay: options.delay,+ }),+ ...(notBefore && { notBefore }),+ ...(deduplicationId && { deduplicationId }),+ ...(options?.retries !== undefined && { retries: options.retries }),+ ...(options?.flowControl && { flowControl: options.flowControl }),+ ...(opts?.batch && options?.queue && { queueName: options.queue }),+ };+}++export function buildReplayRequest(+ job: {+ name: string;+ payload: Prisma.JsonValue;+ options: Prisma.JsonValue | null;+ createdAt: Date;+ scheduledFor: Date | null;+ },+ now: Date = new Date(),+) {+ const options = (job.options ?? {}) as JobReplayOptions;++ const notBefore =+ job.scheduledFor && job.scheduledFor > now+ ? Math.floor(job.scheduledFor.getTime() / 1000)+ : undefined;++ return buildQStashJobRequest(+ {+ name: job.name,+ payload: job.payload,+ options,+ },+ {+ dispatchedAt: job.createdAt.toISOString(),+ batch: true,+ notBefore,+ },+ );+}++export function isPublishSuccess(+ response: unknown,+): response is { messageId: string } {+ return (+ typeof response === "object" &&+ response !== null &&+ "messageId" in response &&+ typeof response.messageId === "string"+ );+}++// Persist jobs that could not be published to QStash. The+// /api/cron/queue/retry cron republishes them and deletes the rows on success.+async function persistBackgroundJobs(inputs: DispatchJobInput[]) {+ const jobs = inputs.map(({ name, payload, options }) => {+ let scheduledFor: Date | null = null;++ if (options?.notBefore) {+ scheduledFor = new Date(options.notBefore * 1000);+ } else if (typeof options?.delay === "number") {+ scheduledFor = new Date(Date.now() + options.delay * 1000);+ }++ const replayOptions = {+ ...(options?.deduplicationId && {+ deduplicationId: options.deduplicationId,+ }),+ ...(options?.retries !== undefined && { retries: options.retries }),+ ...(options?.queue && { queue: options.queue }),+ ...(options?.flowControl && { flowControl: options.flowControl }),+ ...(options?.label && { label: options.label }),+ };++ return {+ id: createId({ prefix: "job_" }),+ name,+ payload: payload as Prisma.InputJsonValue,+ options: replayOptions as Prisma.InputJsonValue,+ scheduledFor,+ };+ });++ await prisma.job.createMany({+ data: jobs,+ });++ return jobs;+}++async function deferJobs(inputs: DispatchJobInput[]) {+ const backgroundJobs = await persistBackgroundJobs(inputs);++ return backgroundJobs.map((backgroundJob) => ({+ status: "deferred" as const,+ backgroundJobId: backgroundJob.id,+ }));+}++async function publishJobsToQStash(inputs: DispatchJobInput[]) {+ if (inputs.length === 0) {+ return [];+ }++ if (inputs.length === 1) {+ const input = inputs[0];+ const request = buildQStashJobRequest(input);++ const response = await withQStashRetry(async () => {+ if (input.options?.queue) {+ return qstash+ .queue({ queueName: input.options.queue })+ .enqueueJSON(request);+ }++ return qstash.publishJSON(request);+ });++ return [response];+ }++ const requests = inputs.map((input) =>+ buildQStashJobRequest(input, { batch: true }),+ );++ return withQStashRetry(() => qstash.batchJSON(requests));+}++async function dispatchJobs(+ inputs: DispatchJobInput[],+): Promise<DispatchBatchResult> {+ if (inputs.length === 0) {+ return {+ published: 0,+ deferred: 0,+ failed: 0,+ results: [],+ };+ }++ const results: DispatchResult[] = [];+ let published = 0;+ let deferred = 0;+ let failed = 0;+ const isSingleDispatch = inputs.length === 1;++ for (const inputChunk of chunk(inputs, QSTASH_BATCH_CHUNK_SIZE)) {+ const jobName = inputChunk[0].name;++ try {+ const responses = await publishJobsToQStash(inputChunk);+ const failedInputs: DispatchJobInput[] = [];++ for (let index = 0; index < inputChunk.length; index++) {+ const input = inputChunk[index];+ const response = responses[index];++ if (isPublishSuccess(response)) {+ published++;+ results.push({+ status: "published",+ messageId: response.messageId,+ });++ if (isSingleDispatch) {+ console.log(`[jobs:${input.name}] published`, {+ jobName: input.name,+ label: buildJobLabel(input.name, input.options?.label),+ messageId: response.messageId,+ queue: input.options?.queue,+ delay: input.options?.delay,+ notBefore: input.options?.notBefore,+ });+ }++ continue;+ }++ failedInputs.push(input);+ }++ if (failedInputs.length > 0) {+ logger.error("jobs.publish_failed", {+ jobName,+ jobCount: failedInputs.length,+ batch: !isSingleDispatch,+ });++ try {+ const deferredResults = await deferJobs(failedInputs);+ deferred += deferredResults.length;+ results.push(...deferredResults);+ } catch (error) {+ logger.error("jobs.dispatch_lost", {+ jobName,+ jobCount: failedInputs.length,+ ...toErrorFields(error),+ });++ failed += failedInputs.length;+ await logger.flush();+ throw error;+ }+ }++ if (!isSingleDispatch) {+ console.log(`[jobs:${jobName}] batch published`, {+ jobName,+ chunkSize: inputChunk.length,+ published: inputChunk.length - failedInputs.length,+ deferred: failedInputs.length,+ });+ }+ } catch (error) {+ logger.error("jobs.publish_failed", {+ jobName,+ jobCount: inputChunk.length,+ batch: !isSingleDispatch,+ ...toErrorFields(error),+ });++ try {+ const deferredResults = await deferJobs(inputChunk);+ deferred += deferredResults.length;+ results.push(...deferredResults);+ } catch (persistError) {+ logger.error("jobs.dispatch_lost", {+ jobName,+ jobCount: inputChunk.length,+ ...toErrorFields(persistError),+ });++ failed += inputChunk.length;+ await logger.flush();+ throw persistError;+ }+ }+ }++ await logger.flush();++ return {+ published,+ deferred,+ failed,+ results,+ };+}++export function defineJob<TSchema extends z.ZodType>({+ name,+ schema,+ defaults,+ handle,+}: {+ name: string;+ schema: TSchema;+ defaults?: JobDefaults;+ handle: (payload: z.infer<TSchema>) => Promise<void>;+}) {+ jobNameSchema.parse(name);++ return {+ name,++ execute: async (payload: unknown) => {+ const parsed = schema.parse(payload);+ await handle(parsed);+ },++ dispatch: async (+ payload: z.infer<TSchema>,+ options?: JobDispatchOptions,+ ) => {+ const { results } = await dispatchJobs([+ {+ name,+ payload,+ options: { ...defaults, ...options },+ },+ ]);+ return results[0];+ },++ dispatchBatch: (+ payloads: z.infer<TSchema>[],+ getOptions?: (+ payload: z.infer<TSchema>,+ index: number,+ ) => JobDispatchOptions | undefined,+ ) =>+ dispatchJobs(+ payloads.map((payload, index) => ({+ name,+ payload,+ options: { ...defaults, ...getOptions?.(payload, index) },+ })),+ ),+ };+}++export type JobDefinition = ReturnType<typeof defineJob<z.ZodType>>;diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.tsnew file mode 100644index 0000000000..85fd5ef652--- /dev/null+++ b/apps/web/lib/jobs/registry.ts@@ -0,0 +1,35 @@+import type { JobDefinition } from "./index";++// Add handlers/{name}-job.ts. Set defineJob({ name: "{name}-job" }).+// Each entry uses a static import() so webpack code-splits handlers into separate chunks.+const jobLoaders = {+ "folder-deleted-job": () =>+ import("./handlers/folder-deleted-job").then((m) => m.folderDeletedJob),++ "partner-tag-deleted-job": () =>+ import("./handlers/partner-tag-deleted-job").then(+ (m) => m.partnerTagDeletedJob,+ ),+} as const satisfies Record<string, () => Promise<JobDefinition>>;++const jobCache = new Map<string, JobDefinition>();++export async function loadJob(+ name: string,+): Promise<JobDefinition | undefined> {+ const cached = jobCache.get(name);+ if (cached) return cached;++ const loader = jobLoaders[name as keyof typeof jobLoaders];+ if (!loader) return undefined;++ const job = await loader();+ if (job.name !== name) {+ throw new Error(`Job name mismatch: ${job.name} !== ${name}`);+ }++ jobCache.set(name, job);+ return job;+}++export const registeredJobNames = Object.keys(jobLoaders);diff --git a/apps/web/prisma/schema/job.prisma b/apps/web/prisma/schema/job.prismanew file mode 100644index 0000000000..dbc55bae01--- /dev/null+++ b/apps/web/prisma/schema/job.prisma@@ -0,0 +1,15 @@+// Background jobs that could not be published to QStash (e.g. QStash outage).+// The /api/cron/queue/retry cron republishes these and deletes them on success.+model Job {+ id String @id+ name String+ payload Json+ options Json? // dispatch options replayed verbatim: deduplicationId, retries, queue, flowControl, label+ scheduledFor DateTime? // absolute time the job should run (derived from delay/notBefore at dispatch time)+ attempts Int @default(0)+ lastError String? @db.Text+ createdAt DateTime @default(now())+ updatedAt DateTime @updatedAt++ @@index([createdAt])+}diff --git a/apps/web/vercel.json b/apps/web/vercel.jsonindex ab9891b717..91c88525c6 100644--- a/apps/web/vercel.json+++ b/apps/web/vercel.json@@ -83,6 +83,10 @@{"path": "/api/cron/webhooks/sync-click-workspaces","schedule": "*/5 * * * *"+ },+ {+ "path": "/api/cron/queue/retry",+ "schedule": "* * * * *"}],"functions": {