Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Route QStash workflows through the job queue for durable publish and retry.

Kiran K5d16ce9a5c13
diff --git a/apps/web/app/(ee)/api/cron/queue/retry/route.ts b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
index 1c1af54078..ccf2d2d520 100644
--- a/apps/web/app/(ee)/api/cron/queue/retry/route.ts
+++ b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
@@ -1,16 +1,10 @@
-import { logger } from "@/lib/axiom/server";
-import { qstash } from "@/lib/cron";
import { withCron } from "@/lib/cron/with-cron";
-import { buildReplayRequest, isPublishSuccess } from "@/lib/jobs";
-import { prisma } from "@/lib/prisma";
+import { publishPendingJobs } from "@/lib/jobs/publish-jobs";
import { redis } from "@/lib/upstash/redis";
import { logAndRespond } from "../../utils";
export const dynamic = "force-dynamic";
-const MAX_ATTEMPTS = 10;
-const BATCH_SIZE = 100;
-
// Lock for the cron job. TTL must be ≥ cron maxDuration (600s in vercel.json)
// so the lock cannot expire while a run is still alive and allow a concurrent
// minute-cron invocation
@@ -18,7 +12,7 @@ const LOCK_KEY = "lock:queue-retry";
const LOCK_TTL_SECONDS = 600;
// GET /api/cron/queue/retry – republish background jobs that failed to
-// publish to QStash at dispatch time; rows are deleted on successful publish
+// publish to QStash at dispatch time; rows are marked published on success
export const GET = withCron(async () => {
const acquired = await redis.set(LOCK_KEY, "1", {
nx: true,
@@ -32,135 +26,20 @@ export const GET = withCron(async () => {
}
try {
- const jobs = await prisma.job.findMany({
- where: {
- attempts: {
- lt: MAX_ATTEMPTS,
- },
- },
- orderBy: {
- createdAt: "asc",
- },
- take: BATCH_SIZE,
- });
+ const { attempted, published, failed } = await publishPendingJobs();
- if (jobs.length === 0) {
+ if (attempted === 0) {
return logAndRespond("No background jobs to retry.");
}
- const now = new Date();
- const entries = jobs.map((job) => buildReplayRequest(job, now));
- const jobIds = jobs.map(({ id }) => id);
- let responses: unknown[];
-
- try {
- responses = await qstash.batchJSON(entries);
- } catch (error) {
- const errorMessage =
- error instanceof Error ? error.message : String(error);
-
- await prisma.job.updateMany({
- where: {
- id: {
- in: jobIds,
- },
- },
- data: {
- attempts: {
- increment: 1,
- },
- lastError: errorMessage.slice(0, 1000),
- },
- });
-
- logger.error("jobs.retry_failed", {
- jobCount: jobs.length,
- errorMessage,
- });
-
- // Jobs that just ran out of attempts are excluded from future runs by the
- // MAX_ATTEMPTS filter and need manual intervention
- const exhaustedJobs = jobs.filter(
- (backgroundJob) => backgroundJob.attempts + 1 >= MAX_ATTEMPTS,
- );
-
- if (exhaustedJobs.length > 0) {
- logger.error("jobs.retry_exhausted", {
- jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
- });
- }
-
- await logger.flush();
-
+ if (failed === 0) {
return logAndRespond(
- `Failed to republish ${jobs.length} background jobs to QStash.`,
+ `Republished ${published} background jobs to QStash.`,
);
}
- const publishedJobIds: string[] = [];
- const failedJobs: typeof jobs = [];
-
- jobs.forEach((job, index) => {
- if (isPublishSuccess(responses[index])) {
- publishedJobIds.push(job.id);
- } else {
- failedJobs.push(job);
- }
- });
-
- if (publishedJobIds.length > 0) {
- await prisma.job.deleteMany({
- where: {
- id: {
- in: publishedJobIds,
- },
- },
- });
- }
-
- if (failedJobs.length === 0) {
- return logAndRespond(
- `Republished ${jobs.length} background jobs to QStash.`,
- );
- }
-
- const errorMessage = "QStash batch publish did not return a messageId";
-
- await prisma.job.updateMany({
- where: {
- id: {
- in: failedJobs.map(({ id }) => id),
- },
- },
- data: {
- attempts: {
- increment: 1,
- },
- lastError: errorMessage,
- },
- });
-
- logger.error("jobs.retry_failed", {
- jobCount: failedJobs.length,
- errorMessage,
- });
-
- // Jobs that just ran out of attempts are excluded from future runs by the
- // MAX_ATTEMPTS filter and need manual intervention
- const exhaustedJobs = failedJobs.filter(
- (backgroundJob) => backgroundJob.attempts + 1 >= MAX_ATTEMPTS,
- );
-
- if (exhaustedJobs.length > 0) {
- logger.error("jobs.retry_exhausted", {
- jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
- });
- }
-
- await logger.flush();
-
return logAndRespond(
- `Republished ${publishedJobIds.length} background jobs to QStash; failed to republish ${failedJobs.length}.`,
+ `Republished ${published} background jobs to QStash; failed to republish ${failed}.`,
);
} finally {
await redis.del(LOCK_KEY);
diff --git a/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts b/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts
index 8b6a7a2465..5ead74ea13 100644
--- a/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts
+++ b/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts
@@ -1,7 +1,7 @@
import { DubApiError } from "@/lib/api/errors";
import { parseRequestBody } from "@/lib/api/utils";
import { withWorkspace } from "@/lib/auth";
-import { triggerQStashWorkflow } from "@/lib/cron/qstash-workflow";
+import { dispatchJobs } from "@/lib/jobs/publish-jobs";
import { prisma } from "@/lib/prisma";
import { ACME_PROGRAM_ID, nanoid } from "@dub/utils";
import { NextResponse } from "next/server";
@@ -52,17 +52,18 @@ export const POST = withWorkspace(
const userId = `e2e-merge-${nanoid()}`;
- const res = await triggerQStashWorkflow({
- workflowType: "merge-partner-accounts",
- workflowLabel: userId,
- body: {
+ const res = await dispatchJobs({
+ name: "mergePartnerAccounts",
+ payload: {
userId,
sourceEmail,
targetEmail,
},
- flowControl: {
- key: userId,
- parallelism: 1,
+ options: {
+ flowControl: {
+ key: userId,
+ parallelism: 1,
+ },
},
});
diff --git a/apps/web/app/(ee)/api/partner-profile/referrals/route.ts b/apps/web/app/(ee)/api/partner-profile/referrals/route.ts
index 7c1c05b700..592513b997 100644
--- a/apps/web/app/(ee)/api/partner-profile/referrals/route.ts
+++ b/apps/web/app/(ee)/api/partner-profile/referrals/route.ts
@@ -2,7 +2,7 @@ import { createId } from "@/lib/api/create-id";
import { DubApiError } from "@/lib/api/errors";
import { obfuscateCustomerEmail } from "@/lib/api/partner-profile/obfuscate-customer-email";
import { withPartnerProfile } from "@/lib/auth/partner";
-import { triggerQStashWorkflow } from "@/lib/cron/qstash-workflow";
+import { dispatchJobs } from "@/lib/jobs/publish-jobs";
import {
getNetworkReferralsQuerySchema,
networkReferralSchema,
@@ -56,10 +56,10 @@ export const GET = withPartnerProfile(async ({ partner, searchParams }) => {
console.log(
"Program enrollment created in the last 1 min, most likely it's a new partner",
);
- await triggerQStashWorkflow({
- workflowType: "partner-approved",
- workflowLabel: partner.id,
- body: {
+
+ await dispatchJobs({
+ name: "partnerApproved",
+ payload: {
programId: NETWORK_PROGRAM_ID,
partnerId: partner.id,
userId: NETWORK_USER_ID,
diff --git a/apps/web/lib/actions/partners/bulk-approve-partners.ts b/apps/web/lib/actions/partners/bulk-approve-partners.ts
index 0466c3562d..6acc58cb81 100644
--- a/apps/web/lib/actions/partners/bulk-approve-partners.ts
+++ b/apps/web/lib/actions/partners/bulk-approve-partners.ts
@@ -4,7 +4,7 @@ import { trackActivityLog } from "@/lib/api/activity-log/track-activity-log";
import { getGroupOrThrow } from "@/lib/api/groups/get-group-or-throw";
import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw";
import { trackApplicationEvents } from "@/lib/application-events/update-application-event";
-import { triggerQStashWorkflow } from "@/lib/cron/qstash-workflow";
+import { dispatchJobs } from "@/lib/jobs/publish-jobs";
import { throwIfPartnersLimitExceeded } from "@/lib/partners/throw-if-partners-limit-exceeded";
import { prisma } from "@/lib/prisma";
import { bulkApprovePartnersSchema } from "@/lib/zod/schemas/partners";
@@ -142,15 +142,17 @@ export const bulkApprovePartnersAction = authActionClient
})),
),
- triggerQStashWorkflow(
+ dispatchJobs(
updatedEnrollments.map(({ partnerId, programId }) => ({
- workflowType: "partner-approved",
- workflowLabel: partnerId,
- body: {
+ name: "partnerApproved",
+ payload: {
programId,
partnerId,
userId: user.id,
},
+ options: {
+ label: partnerId,
+ },
})),
),
diff --git a/apps/web/lib/actions/partners/merge-partner-accounts.ts b/apps/web/lib/actions/partners/merge-partner-accounts.ts
index 8fb1315f59..5d1cf7aa6a 100644
--- a/apps/web/lib/actions/partners/merge-partner-accounts.ts
+++ b/apps/web/lib/actions/partners/merge-partner-accounts.ts
@@ -1,7 +1,7 @@
"use server";
import { generateOTP } from "@/lib/auth/utils";
-import { triggerQStashWorkflow } from "@/lib/cron/qstash-workflow";
+import { dispatchJobs } from "@/lib/jobs/publish-jobs";
import { prisma } from "@/lib/prisma";
import { ratelimit, redis } from "@/lib/upstash";
import { emailSchema } from "@/lib/zod/schemas/auth";
@@ -340,17 +340,18 @@ const mergeAccounts = async ({ userId }: { userId: string }) => {
const { sourceEmail, targetEmail } = accounts;
- await triggerQStashWorkflow({
- workflowType: "merge-partner-accounts",
- workflowLabel: userId,
- body: {
+ await dispatchJobs({
+ name: "mergePartnerAccounts",
+ payload: {
userId,
sourceEmail,
targetEmail,
},
- flowControl: {
- key: userId,
- parallelism: 1,
+ options: {
+ flowControl: {
+ key: userId,
+ parallelism: 1,
+ },
},
});
};
diff --git a/apps/web/lib/api/partners/applications/approve-partner.ts b/apps/web/lib/api/partners/applications/approve-partner.ts
index 2509e3fc1d..a8271af01e 100644
--- a/apps/web/lib/api/partners/applications/approve-partner.ts
+++ b/apps/web/lib/api/partners/applications/approve-partner.ts
@@ -1,9 +1,9 @@
import { trackApplicationEvents } from "@/lib/application-events/update-application-event";
+import { dispatchJobs } from "@/lib/jobs/publish-jobs";
import { prisma } from "@/lib/prisma";
import { ProgramEnrollmentStatus } from "@prisma/client";
import { waitUntil } from "@vercel/functions";
import * as z from "zod/v4";
-import { triggerQStashWorkflow } from "../../../cron/qstash-workflow";
import { throwIfPartnersLimitExceeded } from "../../../partners/throw-if-partners-limit-exceeded";
import { approvePartnerSchema } from "../../../zod/schemas/partners";
import { trackActivityLog } from "../../activity-log/track-activity-log";
@@ -152,14 +152,16 @@ export async function approvePartner({
partnerIds: [partnerId],
}),
- triggerQStashWorkflow({
- workflowType: "partner-approved",
- workflowLabel: partnerId,
- body: {
+ dispatchJobs({
+ name: "partnerApproved",
+ payload: {
programId,
partnerId,
userId,
},
+ options: {
+ label: partnerId,
+ },
}),
]),
);
diff --git a/apps/web/lib/cron/qstash-workflow.ts b/apps/web/lib/cron/qstash-workflow.ts
index a1023c9d0b..1aa1292085 100644
--- a/apps/web/lib/cron/qstash-workflow.ts
+++ b/apps/web/lib/cron/qstash-workflow.ts
@@ -1,128 +1,128 @@
-import { logger, toErrorFields } from "@/lib/axiom/server";
-import { APP_DOMAIN, pluralize } from "@dub/utils";
-import { FlowControl } from "@upstash/qstash";
-import { Client } from "@upstash/workflow";
-
-const client = new Client({
- baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",
- token: process.env.QSTASH_TOKEN || "",
- ...(process.env.VERCEL_ENV === "preview" && {
- headers: {
- "x-vercel-protection-bypass":
- process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",
- },
- }),
-});
-
-type WorkflowType =
- | "partner-approved"
- | "create-partner-commission"
- | "merge-partner-accounts";
-
-interface QStashWorkflow {
- workflowType: WorkflowType;
- workflowLabel: string;
- body: Record<string, unknown>;
- flowControl?: FlowControl;
-}
-
-// Run workflows
-export async function triggerQStashWorkflow(
- input: QStashWorkflow | QStashWorkflow[],
-) {
- const workflows = Array.isArray(input) ? input : [input];
- const maxRetries = 3;
-
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
- try {
- const response = await client.trigger(
- workflows.map((workflow) => ({
- url: `${APP_DOMAIN}/api/workflows/${workflow.workflowType}`,
- body: workflow.body,
- label: workflow.workflowLabel,
- retries: 5,
- flowControl: workflow.flowControl ?? {
- key: workflow.workflowType,
- parallelism: 15,
- },
- })),
- );
-
- console.log(
- `${response.length} QStash ${pluralize("workflow", response.length)} triggered`,
- response,
- );
-
- return response;
- } catch (error) {
- console.error("QStash workflow trigger failed", { error, workflows });
-
- if (attempt < maxRetries) {
- await new Promise((resolve) =>
- setTimeout(resolve, 1000 * Math.pow(2, attempt)),
- );
- continue;
- }
-
- for (const workflow of workflows) {
- const { correlation } = getWorkflowConfig(workflow);
-
- logger.error("workflow.trigger_failed", {
- service: "qstash",
- event: "workflow.trigger_failed",
- workflowType: workflow.workflowType,
- error: toErrorFields(error),
- correlation,
- });
- }
-
- await logger.flush();
-
- return null;
- }
- }
-}
-
-export function getWorkflowConfig({
- workflowType,
- body,
-}: Omit<QStashWorkflow, "workflowLabel">): {
- correlation: Record<string, unknown>;
-} {
- switch (workflowType) {
- case "partner-approved":
- return {
- correlation: {
- programId: body.programId,
- partnerId: body.partnerId,
- userId: body.userId,
- },
- };
-
- case "create-partner-commission": {
- return {
- correlation: {
- programId: body.programId,
- partnerId: body.partnerId,
- customerId: body.customerId,
- bountySubmissionId: body.bountySubmissionId,
- },
- };
- }
-
- case "merge-partner-accounts": {
- return {
- correlation: {
- userId: body.userId,
- sourceEmail: body.sourceEmail,
- targetEmail: body.targetEmail,
- },
- };
- }
-
- default:
- return {
- correlation: {},
- };
- }
-}
+// import { logger, toErrorFields } from "@/lib/axiom/server";
+// import { APP_DOMAIN, pluralize } from "@dub/utils";
+// import { FlowControl } from "@upstash/qstash";
+// import { Client } from "@upstash/workflow";
+
+// export const workflow = new Client({
+// baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",
+// token: process.env.QSTASH_TOKEN || "",
+// ...(process.env.VERCEL_ENV === "preview" && {
+// headers: {
+// "x-vercel-protection-bypass":
+// process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",
+// },
+// }),
+// });
+
+// type WorkflowType =
+// | "partner-approved"
+// | "create-partner-commission"
+// | "merge-partner-accounts";
+
+// interface QStashWorkflow {
+// workflowType: WorkflowType;
+// workflowLabel: string;
+// body: Record<string, unknown>;
+// flowControl?: FlowControl;
+// }
+
+// // Run workflows
+// export async function triggerQStashWorkflow(
+// input: QStashWorkflow | QStashWorkflow[],
+// ) {
+// const workflows = Array.isArray(input) ? input : [input];
+// const maxRetries = 3;
+
+// for (let attempt = 0; attempt <= maxRetries; attempt++) {
+// try {
+// const response = await workflow.trigger(
+// workflows.map((item) => ({
+// url: `${APP_DOMAIN}/api/workflows/${item.workflowType}`,
+// body: item.body,
+// label: item.workflowLabel,
+// retries: 5,
+// flowControl: item.flowControl ?? {
+// key: item.workflowType,
+// parallelism: 15,
+// },
+// })),
+// );
+
+// console.log(
+// `${response.length} QStash ${pluralize("workflow", response.length)} triggered`,
+// response,
+// );
+
+// return response;
+// } catch (error) {
+// console.error("QStash workflow trigger failed", { error, workflows });
+
+// if (attempt < maxRetries) {
+// await new Promise((resolve) =>
+// setTimeout(resolve, 1000 * Math.pow(2, attempt)),
+// );
+// continue;
+// }
+
+// for (const workflow of workflows) {
+// const { correlation } = getWorkflowConfig(workflow);
+
+// logger.error("workflow.trigger_failed", {
+// service: "qstash",
+// event: "workflow.trigger_failed",
+// workflowType: workflow.workflowType,
+// error: toErrorFields(error),
+// correlation,
+// });
+// }
+
+// await logger.flush();
+
+// return null;
+// }
+// }
+// }
+
+// export function getWorkflowConfig({
+// workflowType,
+// body,
+// }: Omit<QStashWorkflow, "workflowLabel">): {
+// correlation: Record<string, unknown>;
+// } {
+// switch (workflowType) {
+// case "partner-approved":
+// return {
+// correlation: {
+// programId: body.programId,
+// partnerId: body.partnerId,
+// userId: body.userId,
+// },
+// };
+
+// case "create-partner-commission": {
+// return {
+// correlation: {
+// programId: body.programId,
+// partnerId: body.partnerId,
+// customerId: body.customerId,
+// bountySubmissionId: body.bountySubmissionId,
+// },
+// };
+// }
+
+// case "merge-partner-accounts": {
+// return {
+// correlation: {
+// userId: body.userId,
+// sourceEmail: body.sourceEmail,
+// targetEmail: body.targetEmail,
+// },
+// };
+// }
+
+// default:
+// return {
+// correlation: {},
+// };
+// }
+// }
diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.ts
index 4e1afd59d5..c0a1efd361 100644
--- a/apps/web/lib/jobs/index.ts
+++ b/apps/web/lib/jobs/index.ts
@@ -154,23 +154,14 @@ function buildQStashJobRequest(
};
}
-export function buildReplayRequest(
- job: {
- name: string;
- payload: Prisma.JsonValue;
- options: Prisma.JsonValue | null;
- createdAt: Date;
- scheduledFor: Date | null;
- },
- now: Date = new Date(),
-) {
+export function buildReplayRequest(job: {
+ name: string;
+ payload: Prisma.JsonValue;
+ options: Prisma.JsonValue | null;
+ createdAt: Date;
+}) {
const options = (job.options ?? {}) as JobReplayOptions;
- const notBefore =
- job.scheduledFor && job.scheduledFor > now
- ? Math.floor(job.scheduledFor.getTime() / 1000)
- : undefined;
-
return buildQStashJobRequest(
{
name: job.name,
@@ -180,7 +171,6 @@ export function buildReplayRequest(
{
dispatchedAt: job.createdAt.toISOString(),
batch: true,
- notBefore,
},
);
}
@@ -200,12 +190,12 @@ export function isPublishSuccess(
// /api/cron/queue/retry cron republishes them and deletes the rows on success.
async function persistBackgroundJobs(inputs: DispatchJobInput[]) {
const jobs = inputs.map(({ name, payload, options }) => {
- let scheduledFor: Date | null = null;
+ let scheduledAt = new Date();
if (options?.notBefore) {
- scheduledFor = new Date(options.notBefore * 1000);
+ scheduledAt = new Date(options.notBefore * 1000);
} else if (typeof options?.delay === "number") {
- scheduledFor = new Date(Date.now() + options.delay * 1000);
+ scheduledAt = new Date(Date.now() + options.delay * 1000);
}
const replayOptions = {
@@ -223,7 +213,7 @@ async function persistBackgroundJobs(inputs: DispatchJobInput[]) {
name,
payload: payload as Prisma.InputJsonValue,
options: replayOptions as Prisma.InputJsonValue,
- scheduledFor,
+ scheduledAt,
};
});
diff --git a/apps/web/lib/jobs/publish-jobs.ts b/apps/web/lib/jobs/publish-jobs.ts
new file mode 100644
index 0000000000..85c9c1df66
--- /dev/null
+++ b/apps/web/lib/jobs/publish-jobs.ts
@@ -0,0 +1,177 @@
+import { logger } from "@/lib/axiom/server";
+import { pluck } from "@dub/utils";
+import { JobStatus, Prisma } from "@prisma/client";
+import { createId } from "../api/create-id";
+import { prisma } from "../prisma";
+import { QStashJobPublisher } from "./publishers/qstash-job-publisher";
+import { QStashWorkflowPublisher } from "./publishers/qstash-workflow-publisher";
+import type { QStashJobOptions } from "./publishers/types";
+
+const MAX_ATTEMPTS = 10;
+const BATCH_SIZE = 100;
+
+type DispatchJobInput = {
+ name: string;
+ payload: unknown;
+ options?: QStashJobOptions;
+};
+
+export type PublishPendingJobsResult = {
+ attempted: number;
+ published: number;
+ failed: number;
+};
+
+const qstashJobPublisher = new QStashJobPublisher();
+const qstashWorkflowPublisher = new QStashWorkflowPublisher();
+
+export async function dispatchJobs(
+ input: DispatchJobInput | DispatchJobInput[],
+) {
+ const jobs = Array.isArray(input) ? input : [input];
+
+ if (jobs.length === 0) {
+ return;
+ }
+
+ const data = jobs.map((job) => ({
+ id: createId({ prefix: "job_" }),
+ name: job.name,
+ scheduledAt: new Date(),
+ payload: job.payload as Prisma.InputJsonValue,
+ ...(job.options && {
+ options: job.options as Prisma.InputJsonValue,
+ }),
+ }));
+
+ // Durably persist the jobs in the database
+ await prisma.job.createMany({
+ skipDuplicates: true,
+ data: data,
+ });
+
+ const jobIds = pluck(data, "id");
+
+ // Best effort publish, if the job is not published, it will be published on the next cron run
+ await publishPendingJobs(jobIds);
+}
+
+export async function publishPendingJobs(
+ jobIds?: string[],
+): Promise<PublishPendingJobsResult> {
+ const emptyResult: PublishPendingJobsResult = {
+ attempted: 0,
+ published: 0,
+ failed: 0,
+ };
+
+ if (jobIds && jobIds.length === 0) {
+ return emptyResult;
+ }
+
+ const isScanMode = jobIds === undefined;
+
+ const jobs = await prisma.job.findMany({
+ where: {
+ ...(jobIds && {
+ id: {
+ in: jobIds,
+ },
+ }),
+ scheduledAt: {
+ lte: new Date(),
+ },
+ status: {
+ in: [JobStatus.pending, JobStatus.failed],
+ },
+ attempts: {
+ lt: MAX_ATTEMPTS,
+ },
+ },
+ ...(isScanMode && {
+ orderBy: {
+ createdAt: "asc" as const,
+ },
+ take: BATCH_SIZE,
+ }),
+ });
+
+ if (jobs.length === 0) {
+ return emptyResult;
+ }
+
+ const workflowJobs = jobs.filter(QStashWorkflowPublisher.isWorkflowJob);
+ const qstashJobs = jobs.filter(
+ (job) => !QStashWorkflowPublisher.isWorkflowJob(job),
+ );
+
+ const results = (
+ await Promise.all([
+ qstashWorkflowPublisher.send(workflowJobs),
+ qstashJobPublisher.send(qstashJobs),
+ ])
+ ).flat();
+
+ const publishedIds = results
+ .filter((result) => result.status === JobStatus.published)
+ .map((result) => result.id);
+
+ const failedResults = results.filter(
+ (result) => result.status === JobStatus.failed,
+ );
+
+ // Job has been published successfully
+ if (publishedIds.length > 0) {
+ await prisma.job.updateMany({
+ where: {
+ id: {
+ in: publishedIds,
+ },
+ },
+ data: {
+ status: JobStatus.published,
+ },
+ });
+ }
+
+ // Job has failed
+ if (failedResults.length > 0) {
+ await Promise.all(
+ failedResults.map((result) =>
+ prisma.job.update({
+ where: {
+ id: result.id,
+ },
+ data: {
+ status: JobStatus.failed,
+ lastError: result.lastError,
+ attempts: {
+ increment: 1,
+ },
+ },
+ }),
+ ),
+ );
+
+ // Jobs that just ran out of attempts are excluded from future runs by the
+ // MAX_ATTEMPTS filter and need manual intervention
+ const exhaustedJobs = jobs.filter(
+ (job) =>
+ failedResults.some((result) => result.id === job.id) &&
+ job.attempts + 1 >= MAX_ATTEMPTS,
+ );
+
+ if (exhaustedJobs.length > 0) {
+ logger.error("jobs.retry_exhausted", {
+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
+ });
+ await logger.flush();
+ }
+ }
+
+ return {
+ attempted: jobs.length,
+ published: publishedIds.length,
+ failed: failedResults.length,
+ };
+}
diff --git a/apps/web/lib/jobs/publishers/qstash-job-publisher.ts b/apps/web/lib/jobs/publishers/qstash-job-publisher.ts
new file mode 100644
index 0000000000..b6497913a5
--- /dev/null
+++ b/apps/web/lib/jobs/publishers/qstash-job-publisher.ts
@@ -0,0 +1,80 @@
+import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
+import { Job, JobStatus } from "@prisma/client";
+import { PublishBatchRequest } from "@upstash/qstash";
+import { qstash } from "../../cron";
+import { isPublishSuccess } from "../index";
+import { JobPublisher, PublishSendResult, QStashJobOptions } from "./types";
+
+const QSTASH_BATCH_CHUNK_SIZE = 100;
+
+const jobPathMap = {
+ welcomeUser: "/api/cron/welcome-user",
+};
+
+export class QStashJobPublisher implements JobPublisher {
+ async send(jobs: Job[]): Promise<PublishSendResult[]> {
+ if (jobs.length === 0) {
+ return [];
+ }
+
+ const results: PublishSendResult[] = [];
+
+ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
+ try {
+ const responses = await qstash.batchJSON(
+ jobChunk.map((job) => this.buildRequest(job)),
+ );
+
+ jobChunk.forEach((job, index) => {
+ if (isPublishSuccess(responses[index])) {
+ results.push({
+ id: job.id,
+ status: JobStatus.published,
+ lastError: null,
+ });
+ } else {
+ results.push({
+ id: job.id,
+ status: JobStatus.failed,
+ lastError: "QStash batch publish did not return a messageId",
+ });
+ }
+ });
+ } catch (error) {
+ const lastError = serializeError(error);
+
+ for (const job of jobChunk) {
+ results.push({
+ id: job.id,
+ status: JobStatus.failed,
+ lastError,
+ });
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private buildRequest(job: Job): PublishBatchRequest<unknown> {
+ let jobPath = jobPathMap[job.name as keyof typeof jobPathMap];
+
+ if (!jobPath) {
+ jobPath = `/api/jobs/process/${job.name}`;
+ }
+
+ const options = (job.options ?? {}) as QStashJobOptions;
+
+ return {
+ url: `${APP_DOMAIN_WITH_NGROK}${jobPath}`,
+ body: job.payload,
+ ...(options.label && { label: options.label }),
+ ...(options.deduplicationId && {
+ deduplicationId: options.deduplicationId,
+ }),
+ ...(options.retries !== undefined && { retries: options.retries }),
+ ...(options.flowControl && { flowControl: options.flowControl }),
+ ...(options.queue && { queueName: options.queue }),
+ };
+ }
+}
diff --git a/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts b/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts
new file mode 100644
index 0000000000..49a646bec2
--- /dev/null
+++ b/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts
@@ -0,0 +1,105 @@
+import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
+import { Job, JobStatus } from "@prisma/client";
+import { Client, TriggerOptions } from "@upstash/workflow";
+import { JobPublisher, PublishSendResult, QStashJobOptions } from "./types";
+
+const workflowClient = new Client({
+ baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",
+ token: process.env.QSTASH_TOKEN || "",
+ ...(process.env.VERCEL_ENV === "preview" && {
+ headers: {
+ "x-vercel-protection-bypass":
+ process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",
+ },
+ }),
+});
+
+const workflowPathMap = {
+ partnerApproved: "/api/workflows/partner-approved",
+ mergePartnerAccounts: "/api/workflows/merge-partner-accounts",
+ createPartnerCommission: "/api/workflows/create-partner-commission",
+};
+
+const QSTASH_BATCH_CHUNK_SIZE = 100;
+
+type WorkflowJob = Job & { name: keyof typeof workflowPathMap };
+
+export class QStashWorkflowPublisher implements JobPublisher {
+ static isWorkflowJob(job: Job): job is WorkflowJob {
+ return job.name in workflowPathMap;
+ }
+
+ async send(jobs: Job[]): Promise<PublishSendResult[]> {
+ const workflowJobs = jobs.filter(QStashWorkflowPublisher.isWorkflowJob);
+
+ if (workflowJobs.length === 0) {
+ return [];
+ }
+
+ const results: PublishSendResult[] = [];
+
+ for (const jobChunk of chunk(workflowJobs, QSTASH_BATCH_CHUNK_SIZE)) {
+ try {
+ const responses = await workflowClient.trigger(
+ jobChunk.map((job) => this.buildRequest(job)),
+ );
+
+ jobChunk.forEach((job, index) => {
+ if (this.isTriggerSuccess(responses[index])) {
+ results.push({
+ id: job.id,
+ status: JobStatus.published,
+ lastError: null,
+ });
+ } else {
+ results.push({
+ id: job.id,
+ status: JobStatus.failed,
+ lastError: "Workflow trigger did not return a workflowRunId",
+ });
+ }
+ });
+ } catch (error) {
+ const lastError = serializeError(error);
+
+ for (const job of jobChunk) {
+ results.push({
+ id: job.id,
+ status: JobStatus.failed,
+ lastError,
+ });
+ }
+ }
+ }
+
+ return results;
+ }
+
+ private buildRequest(job: WorkflowJob): TriggerOptions {
+ const options = (job.options ?? {}) as QStashJobOptions;
+ const workflowPath = workflowPathMap[job.name];
+ const workflowKey = workflowPath.split("/").pop()!;
+
+ return {
+ url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,
+ body: job.payload,
+ retries: options.retries ?? 5,
+ flowControl: options.flowControl ?? {
+ key: workflowKey,
+ parallelism: 15,
+ },
+ ...(options.label && { label: options.label }),
+ };
+ }
+
+ private isTriggerSuccess(
+ response: unknown,
+ ): response is { workflowRunId: string } {
+ return (
+ typeof response === "object" &&
+ response !== null &&
+ "workflowRunId" in response &&
+ typeof response.workflowRunId === "string"
+ );
+ }
+}
diff --git a/apps/web/lib/jobs/publishers/types.ts b/apps/web/lib/jobs/publishers/types.ts
new file mode 100644
index 0000000000..7468d4de7b
--- /dev/null
+++ b/apps/web/lib/jobs/publishers/types.ts
@@ -0,0 +1,19 @@
+import { Job, JobStatus } from "@prisma/client";
+import { PublishRequest } from "@upstash/qstash";
+
+export type QStashJobOptions = Pick<
+ PublishRequest,
+ "label" | "deduplicationId" | "retries" | "flowControl"
+> & {
+ queue?: string;
+};
+
+export type PublishSendResult = {
+ id: string;
+ status: typeof JobStatus.published | typeof JobStatus.failed;
+ lastError: string | null;
+};
+
+export interface JobPublisher {
+ send(jobs: Job[]): Promise<PublishSendResult[]>;
+}
diff --git a/apps/web/lib/partners/queue-partner-commission-creation.ts b/apps/web/lib/partners/queue-partner-commission-creation.ts
index bc0bf49f12..e1e5fff652 100644
--- a/apps/web/lib/partners/queue-partner-commission-creation.ts
+++ b/apps/web/lib/partners/queue-partner-commission-creation.ts
@@ -1,5 +1,5 @@
import { getProgramEnrollmentOrThrow } from "../api/programs/get-program-enrollment-or-throw";
-import { triggerQStashWorkflow } from "../cron/qstash-workflow";
+import { dispatchJobs } from "../jobs/publish-jobs";
import { CreatePartnerCommissionProps } from "../types";
import { constructWebhookPartner } from "./constuct-webhook-partner";
@@ -19,13 +19,15 @@ export const queuePartnerCommissionCreation = async (
const { partner, links, ...programEnrollment } = result;
- await triggerQStashWorkflow({
- workflowType: "create-partner-commission",
- workflowLabel: bountySubmissionId ?? customerId ?? partnerId,
- body: params,
- flowControl: {
- key: partnerId,
- parallelism: 1,
+ await dispatchJobs({
+ name: "createPartnerCommission",
+ payload: params,
+ options: {
+ flowControl: {
+ key: partnerId,
+ parallelism: 1,
+ },
+ label: bountySubmissionId ?? customerId ?? partnerId,
},
});
diff --git a/apps/web/prisma/schema/job.prisma b/apps/web/prisma/schema/job.prisma
index dbc55bae01..bbfc605729 100644
--- a/apps/web/prisma/schema/job.prisma
+++ b/apps/web/prisma/schema/job.prisma
@@ -1,15 +1,20 @@
-// Background jobs that could not be published to QStash (e.g. QStash outage).
-// The /api/cron/queue/retry cron republishes these and deletes them on success.
+enum JobStatus {
+ pending
+ published
+ failed
+}
+
model Job {
- id String @id
- name String
- payload Json
- options Json? // dispatch options replayed verbatim: deduplicationId, retries, queue, flowControl, label
- scheduledFor DateTime? // absolute time the job should run (derived from delay/notBefore at dispatch time)
- attempts Int @default(0)
- lastError String? @db.Text
- createdAt DateTime @default(now())
- updatedAt DateTime @updatedAt
+ id String @id
+ name String
+ payload Json
+ options Json? // dispatch options replayed verbatim: deduplicationId, retries, queue, flowControl, label
+ scheduledAt DateTime @default(now()) // when we should publish the job
+ status JobStatus @default(pending)
+ attempts Int @default(0)
+ lastError String? @db.Text
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
@@index([createdAt])
}