Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Persist workflows only when QStash publish fails, and drop the write-ahead outbox.

Kiran K18dae57ff60f
diff --git a/apps/web/app/(ee)/api/cron/queue/cleanup/route.ts b/apps/web/app/(ee)/api/cron/queue/cleanup/route.ts
deleted file mode 100644
index 921710da79..0000000000
--- a/apps/web/app/(ee)/api/cron/queue/cleanup/route.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import { withCron } from "@/lib/cron/with-cron";
-import { cleanupPublishedJobs } from "@/lib/jobs/cleanup-jobs";
-import { logAndRespond } from "../../utils";
-
-export const dynamic = "force-dynamic";
-export const maxDuration = 600;
-
-// Deletes published Job rows older than 24 hours so the outbox does not grow unbounded.
-// Runs once every day at 02:00:00 AM UTC (0 2 * * *)
-// GET /api/cron/queue/cleanup
-export const GET = withCron(async () => {
- const { deletedCount } = await cleanupPublishedJobs();
-
- if (deletedCount === 0) {
- return logAndRespond("No published jobs older than 24 hours to delete.");
- }
-
- return logAndRespond(
- `Deleted ${deletedCount} published jobs older than 24 hours.`,
- );
-});
diff --git a/apps/web/app/(ee)/api/cron/queue/retry/route.ts b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
index ccf2d2d520..f16bab8611 100644
--- a/apps/web/app/(ee)/api/cron/queue/retry/route.ts
+++ b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
@@ -1,5 +1,6 @@
import { withCron } from "@/lib/cron/with-cron";
-import { publishPendingJobs } from "@/lib/jobs/publish-jobs";
+import { publishPendingDefineJobs } from "@/lib/jobs";
+import { publishPendingWorkflows } from "@/lib/jobs/publish-workflows";
import { redis } from "@/lib/upstash/redis";
import { logAndRespond } from "../../utils";
@@ -11,8 +12,8 @@ export const dynamic = "force-dynamic";
const LOCK_KEY = "lock:queue-retry";
const LOCK_TTL_SECONDS = 600;
-// GET /api/cron/queue/retry – republish background jobs that failed to
-// publish to QStash at dispatch time; rows are marked published on success
+// GET /api/cron/queue/retry – republish workflows / defineJob rows that failed
+// to publish to QStash at dispatch time; rows are deleted on successful publish
export const GET = withCron(async () => {
const acquired = await redis.set(LOCK_KEY, "1", {
nx: true,
@@ -26,7 +27,14 @@ export const GET = withCron(async () => {
}
try {
- const { attempted, published, failed } = await publishPendingJobs();
+ const [workflows, defineJobs] = await Promise.all([
+ publishPendingWorkflows(),
+ publishPendingDefineJobs(),
+ ]);
+
+ const attempted = workflows.attempted + defineJobs.attempted;
+ const published = workflows.published + defineJobs.published;
+ const failed = workflows.failed + defineJobs.failed;
if (attempted === 0) {
return logAndRespond("No background jobs to retry.");
diff --git a/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts b/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts
index d71b7b4330..1830d5ac7d 100644
--- a/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts
+++ b/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts
@@ -1,7 +1,7 @@
import { DubApiError } from "@/lib/api/errors";
import { parseRequestBody } from "@/lib/api/utils";
import { withWorkspace } from "@/lib/auth";
-import { dispatchJobs } from "@/lib/jobs/publish-jobs";
+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";
import { prisma } from "@/lib/prisma";
import { ACME_PROGRAM_ID, nanoid } from "@dub/utils";
import { NextResponse } from "next/server";
@@ -52,7 +52,7 @@ export const POST = withWorkspace(
const userId = `e2e-merge-${nanoid()}`;
- const res = await dispatchJobs({
+ const res = await dispatchWorkflows({
name: "mergePartnerAccounts",
payload: {
userId,
diff --git a/apps/web/app/(ee)/api/partner-profile/referrals/route.ts b/apps/web/app/(ee)/api/partner-profile/referrals/route.ts
index 832e1a264d..ccb2c7e673 100644
--- a/apps/web/app/(ee)/api/partner-profile/referrals/route.ts
+++ b/apps/web/app/(ee)/api/partner-profile/referrals/route.ts
@@ -2,7 +2,7 @@ import { createId } from "@/lib/api/create-id";
import { DubApiError } from "@/lib/api/errors";
import { obfuscateCustomerEmail } from "@/lib/api/partner-profile/obfuscate-customer-email";
import { withPartnerProfile } from "@/lib/auth/partner";
-import { dispatchJobs } from "@/lib/jobs/publish-jobs";
+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";
import {
getNetworkReferralsQuerySchema,
networkReferralSchema,
@@ -57,7 +57,7 @@ export const GET = withPartnerProfile(async ({ partner, searchParams }) => {
"Program enrollment created in the last 1 min, most likely it's a new partner",
);
- await dispatchJobs({
+ await dispatchWorkflows({
name: "partnerApproved",
payload: {
programId: NETWORK_PROGRAM_ID,
diff --git a/apps/web/lib/actions/partners/bulk-approve-partners.ts b/apps/web/lib/actions/partners/bulk-approve-partners.ts
index 822f2786ff..7a10b8b34f 100644
--- a/apps/web/lib/actions/partners/bulk-approve-partners.ts
+++ b/apps/web/lib/actions/partners/bulk-approve-partners.ts
@@ -5,7 +5,7 @@ import { getGroupOrThrow } from "@/lib/api/groups/get-group-or-throw";
import { queuePartnerSearchSync } from "@/lib/api/partners/queue-partner-search-sync";
import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw";
import { trackApplicationEvents } from "@/lib/application-events/update-application-event";
-import { dispatchJobs } from "@/lib/jobs/publish-jobs";
+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";
import { throwIfPartnersLimitExceeded } from "@/lib/partners/throw-if-partners-limit-exceeded";
import { prisma } from "@/lib/prisma";
import { bulkApprovePartnersSchema } from "@/lib/zod/schemas/partners";
@@ -143,7 +143,7 @@ export const bulkApprovePartnersAction = authActionClient
})),
),
- dispatchJobs(
+ dispatchWorkflows(
updatedEnrollments.map(({ partnerId, programId }) => ({
name: "partnerApproved",
payload: {
diff --git a/apps/web/lib/actions/partners/merge-partner-accounts.ts b/apps/web/lib/actions/partners/merge-partner-accounts.ts
index 51526b3f70..cfc9e7f3be 100644
--- a/apps/web/lib/actions/partners/merge-partner-accounts.ts
+++ b/apps/web/lib/actions/partners/merge-partner-accounts.ts
@@ -1,7 +1,7 @@
"use server";
import { generateOTP } from "@/lib/auth/utils";
-import { dispatchJobs } from "@/lib/jobs/publish-jobs";
+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";
import { prisma } from "@/lib/prisma";
import { ratelimit, redis } from "@/lib/upstash";
import { emailSchema } from "@/lib/zod/schemas/auth";
@@ -340,7 +340,7 @@ const mergeAccounts = async ({ userId }: { userId: string }) => {
const { sourceEmail, targetEmail } = accounts;
- await dispatchJobs({
+ await dispatchWorkflows({
name: "mergePartnerAccounts",
payload: {
userId,
diff --git a/apps/web/lib/api/partners/applications/approve-partner.ts b/apps/web/lib/api/partners/applications/approve-partner.ts
index ffd032ceb7..4b2097c120 100644
--- a/apps/web/lib/api/partners/applications/approve-partner.ts
+++ b/apps/web/lib/api/partners/applications/approve-partner.ts
@@ -1,5 +1,5 @@
import { trackApplicationEvents } from "@/lib/application-events/update-application-event";
-import { dispatchJobs } from "@/lib/jobs/publish-jobs";
+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";
import { prisma } from "@/lib/prisma";
import { ProgramEnrollmentStatus } from "@prisma/client";
import { waitUntil } from "@vercel/functions";
@@ -156,7 +156,7 @@ export async function approvePartner({
partnerIds: [partnerId],
}),
- dispatchJobs({
+ dispatchWorkflows({
name: "partnerApproved",
payload: {
programId,
diff --git a/apps/web/lib/jobs/cleanup-jobs.ts b/apps/web/lib/jobs/cleanup-jobs.ts
deleted file mode 100644
index 2171da8e89..0000000000
--- a/apps/web/lib/jobs/cleanup-jobs.ts
+++ /dev/null
@@ -1,52 +0,0 @@
-import { prisma } from "@/lib/prisma";
-import { JobStatus } from "@prisma/client";
-import { subHours } from "date-fns";
-
-const MAX_JOBS_PER_BATCH = 1000;
-const RETENTION_HOURS = 24;
-
-export async function cleanupPublishedJobs(): Promise<{
- deletedCount: number;
-}> {
- const cutoff = subHours(new Date(), RETENTION_HOURS);
- let deletedCount = 0;
-
- while (true) {
- const jobs = await prisma.job.findMany({
- where: {
- status: JobStatus.published,
- createdAt: {
- lt: cutoff,
- },
- },
- select: {
- id: true,
- },
- take: MAX_JOBS_PER_BATCH,
- });
-
- if (jobs.length === 0) {
- break;
- }
-
- const { count } = await prisma.job.deleteMany({
- where: {
- id: {
- in: jobs.map((job) => job.id),
- },
- },
- });
-
- deletedCount += count;
-
- if (jobs.length < MAX_JOBS_PER_BATCH) {
- break;
- }
-
- await new Promise((resolve) => setTimeout(resolve, 500));
- }
-
- return {
- deletedCount,
- };
-}
diff --git a/apps/web/lib/jobs/constants.ts b/apps/web/lib/jobs/constants.ts
new file mode 100644
index 0000000000..0d73eb9cc5
--- /dev/null
+++ b/apps/web/lib/jobs/constants.ts
@@ -0,0 +1,4 @@
+export const QSTASH_BATCH_CHUNK_SIZE = 100;
+export const LAST_ERROR_MAX_LENGTH = 1000;
+export const MAX_JOB_ATTEMPTS = 10;
+export const MAX_JOBS_PER_BATCH = 100;
diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.ts
index 237a41d751..1d09986b5e 100644
--- a/apps/web/lib/jobs/index.ts
+++ b/apps/web/lib/jobs/index.ts
@@ -3,9 +3,15 @@ import { logger } from "@/lib/axiom/server";
import { qstash } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";
-import { Prisma } from "@prisma/client";
+import { JobStatus, Prisma } from "@prisma/client";
import { PublishRequest } from "@upstash/qstash";
import * as z from "zod/v4";
+import {
+ LAST_ERROR_MAX_LENGTH,
+ MAX_JOB_ATTEMPTS,
+ MAX_JOBS_PER_BATCH,
+ QSTASH_BATCH_CHUNK_SIZE,
+} from "./constants";
type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;
@@ -50,8 +56,6 @@ const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;
const QSTASH_PUBLISH_MAX_RETRIES = 3;
-const QSTASH_BATCH_CHUNK_SIZE = 100;
-
export const jobNameSchema = z
.string()
.regex(
@@ -110,18 +114,26 @@ async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {
throw new Error("Failed to publish to QStash.");
}
+type JobReplayOptions = Pick<
+ JobDispatchOptions,
+ "deduplicationId" | "retries" | "queue" | "flowControl" | "label"
+>;
+
function buildQStashJobRequest(
{ name, payload, options }: DispatchJobInput,
opts?: {
+ dispatchedAt?: string;
batch?: boolean;
+ notBefore?: number;
},
) {
const envelope: JobEnvelope = {
name,
payload,
- dispatchedAt: new Date().toISOString(),
+ dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),
};
+ const notBefore = opts?.notBefore ?? options?.notBefore;
const deduplicationId = buildJobDeduplicationId(
name,
options?.deduplicationId,
@@ -131,8 +143,11 @@ function buildQStashJobRequest(
url: getJobsEndpointUrl(name),
body: envelope,
label: buildJobLabel(name, options?.label),
- ...(options?.delay && { delay: options.delay }),
- ...(options?.notBefore && { notBefore: options.notBefore }),
+ ...(options?.delay &&
+ opts?.notBefore === undefined && {
+ delay: options.delay,
+ }),
+ ...(notBefore && { notBefore }),
...(deduplicationId && { deduplicationId }),
...(options?.retries !== undefined && { retries: options.retries }),
...(options?.flowControl && { flowControl: options.flowControl }),
@@ -140,6 +155,38 @@ function buildQStashJobRequest(
};
}
+// Used by /api/cron/queue/retry to republish deferred defineJob rows
+export function buildReplayRequest(
+ job: {
+ name: string;
+ payload: Prisma.JsonValue;
+ options: Prisma.JsonValue | null;
+ createdAt: Date;
+ scheduledAt: Date | null;
+ },
+ now: Date = new Date(),
+) {
+ const options = (job.options ?? {}) as JobReplayOptions;
+
+ const notBefore =
+ job.scheduledAt && job.scheduledAt > now
+ ? Math.floor(job.scheduledAt.getTime() / 1000)
+ : undefined;
+
+ return buildQStashJobRequest(
+ {
+ name: job.name,
+ payload: job.payload,
+ options,
+ },
+ {
+ dispatchedAt: job.createdAt.toISOString(),
+ batch: true,
+ notBefore,
+ },
+ );
+}
+
export function isPublishSuccess(
response: unknown,
): response is { messageId: string } {
@@ -401,3 +448,113 @@ export function defineJob<TSchema extends z.ZodType>({
}
export type JobDefinition = ReturnType<typeof defineJob<z.ZodType>>;
+
+/** Republish deferred defineJob rows that failed to publish at dispatch time. */
+export async function publishPendingDefineJobs(): Promise<{
+ attempted: number;
+ published: number;
+ failed: number;
+}> {
+ const jobs = await prisma.job.findMany({
+ where: {
+ name: {
+ endsWith: "-job",
+ },
+ scheduledAt: {
+ lte: new Date(),
+ },
+ status: {
+ in: [JobStatus.pending, JobStatus.failed],
+ },
+ attempts: {
+ lt: MAX_JOB_ATTEMPTS,
+ },
+ },
+ orderBy: {
+ createdAt: "asc",
+ },
+ take: MAX_JOBS_PER_BATCH,
+ });
+
+ if (jobs.length === 0) {
+ return { attempted: 0, published: 0, failed: 0 };
+ }
+
+ const now = new Date();
+ const publishedIds: string[] = [];
+ const failedByError = new Map<string, string[]>();
+
+ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
+ try {
+ const responses = await qstash.batchJSON(
+ jobChunk.map((job) => buildReplayRequest(job, now)),
+ );
+
+ jobChunk.forEach((job, index) => {
+ if (isPublishSuccess(responses[index])) {
+ publishedIds.push(job.id);
+ } else {
+ const key = "QStash batch publish did not return a messageId";
+ const ids = failedByError.get(key) ?? [];
+ ids.push(job.id);
+ failedByError.set(key, ids);
+ }
+ });
+ } catch (error) {
+ const lastError = (
+ error instanceof Error ? error.message : String(error)
+ ).slice(0, LAST_ERROR_MAX_LENGTH);
+
+ for (const job of jobChunk) {
+ const ids = failedByError.get(lastError) ?? [];
+ ids.push(job.id);
+ failedByError.set(lastError, ids);
+ }
+ }
+ }
+
+ if (publishedIds.length > 0) {
+ await prisma.job.deleteMany({
+ where: {
+ id: {
+ in: publishedIds,
+ },
+ },
+ });
+ }
+
+ for (const [lastError, ids] of failedByError) {
+ await prisma.job.updateMany({
+ where: {
+ id: {
+ in: ids,
+ },
+ },
+ data: {
+ status: JobStatus.failed,
+ lastError,
+ attempts: {
+ increment: 1,
+ },
+ },
+ });
+ }
+
+ const failedIds = new Set([...failedByError.values()].flat());
+ const exhaustedJobs = jobs.filter(
+ (job) => failedIds.has(job.id) && job.attempts + 1 >= MAX_JOB_ATTEMPTS,
+ );
+
+ if (exhaustedJobs.length > 0) {
+ logger.error("jobs.retry_exhausted", {
+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
+ });
+ await logger.flush();
+ }
+
+ return {
+ attempted: jobs.length,
+ published: publishedIds.length,
+ failed: failedIds.size,
+ };
+}
diff --git a/apps/web/lib/jobs/publish-jobs.ts b/apps/web/lib/jobs/publish-jobs.ts
deleted file mode 100644
index eb4e842301..0000000000
--- a/apps/web/lib/jobs/publish-jobs.ts
+++ /dev/null
@@ -1,204 +0,0 @@
-import { logger } from "@/lib/axiom/server";
-import { pluck } from "@dub/utils";
-import { Job, JobStatus, Prisma } from "@prisma/client";
-import { createId } from "../api/create-id";
-import { prisma } from "../prisma";
-import { QStashJobPublisher } from "./publishers/qstash-job-publisher";
-import { QStashWorkflowPublisher } from "./publishers/qstash-workflow-publisher";
-import type { PublishSendResult, QStashJobOptions } from "./publishers/types";
-
-const MAX_ATTEMPTS = 10;
-const MAX_JOBS_PER_BATCH = 100;
-
-type DispatchJobInput = {
- name: string;
- payload: unknown;
- options?: QStashJobOptions;
-};
-
-export type PublishPendingJobsResult = {
- attempted: number;
- published: number;
- failed: number;
-};
-
-const qstashJobPublisher = new QStashJobPublisher();
-const qstashWorkflowPublisher = new QStashWorkflowPublisher();
-
-export async function dispatchJobs(
- input: DispatchJobInput | DispatchJobInput[],
-) {
- const jobs = Array.isArray(input) ? input : [input];
-
- if (jobs.length === 0) {
- return;
- }
-
- const data = jobs.map((job) => {
- const id = createId({ prefix: "job_" });
-
- return {
- id,
- name: job.name,
- scheduledAt: new Date(),
- payload: job.payload as Prisma.InputJsonValue,
- // Default deduplicationId to the job id so best-effort publish and the
- // retry cron cannot deliver the same job twice via QStash
- options: {
- ...job.options,
- deduplicationId: job.options?.deduplicationId ?? id,
- } as Prisma.InputJsonValue,
- };
- });
-
- // Durably persist the jobs in the database
- await prisma.job.createMany({
- skipDuplicates: true,
- data: data,
- });
-
- const jobIds = pluck(data, "id");
-
- // Best effort publish, if the job is not published, it will be published on the next cron run
- await publishPendingJobs(jobIds);
-}
-
-export async function publishPendingJobs(
- jobIds?: string[],
-): Promise<PublishPendingJobsResult> {
- const emptyResult: PublishPendingJobsResult = {
- attempted: 0,
- published: 0,
- failed: 0,
- };
-
- if (jobIds && jobIds.length === 0) {
- return emptyResult;
- }
-
- const isScanMode = jobIds === undefined;
-
- const jobs = await prisma.job.findMany({
- where: {
- ...(jobIds && {
- id: {
- in: jobIds,
- },
- }),
- scheduledAt: {
- lte: new Date(),
- },
- status: {
- in: [JobStatus.pending, JobStatus.failed],
- },
- attempts: {
- lt: MAX_ATTEMPTS,
- },
- },
- ...(isScanMode && {
- orderBy: {
- createdAt: "asc" as const,
- },
- take: MAX_JOBS_PER_BATCH,
- }),
- });
-
- if (jobs.length === 0) {
- return emptyResult;
- }
-
- const workflowJobs = jobs.filter(QStashWorkflowPublisher.isWorkflowJob);
- const qstashJobs = jobs.filter(
- (job) => !QStashWorkflowPublisher.isWorkflowJob(job),
- );
-
- const results = (
- await Promise.all([
- qstashWorkflowPublisher.send(workflowJobs),
- qstashJobPublisher.send(qstashJobs),
- ])
- ).flat();
-
- const publishedResults = results.filter(
- (result) => result.status === JobStatus.published,
- );
-
- const failedResults = results.filter(
- (result) => result.status === JobStatus.failed,
- );
-
- await markAsPublished({ publishedResults });
- await markAsFailed({ failedResults, jobs });
-
- return {
- attempted: jobs.length,
- published: publishedResults.length,
- failed: failedResults.length,
- };
-}
-
-async function markAsPublished({
- publishedResults,
-}: {
- publishedResults: PublishSendResult[];
-}) {
- if (publishedResults.length === 0) {
- return;
- }
-
- await prisma.job.updateMany({
- where: {
- id: {
- in: publishedResults.map((result) => result.id),
- },
- },
- data: {
- status: JobStatus.published,
- },
- });
-}
-
-async function markAsFailed({
- failedResults,
- jobs,
-}: {
- failedResults: PublishSendResult[];
- jobs: Pick<Job, "id" | "name" | "attempts">[];
-}) {
- if (failedResults.length === 0) {
- return;
- }
-
- await Promise.all(
- failedResults.map((result) =>
- prisma.job.update({
- where: {
- id: result.id,
- },
- data: {
- status: JobStatus.failed,
- lastError: result.lastError,
- attempts: {
- increment: 1,
- },
- },
- }),
- ),
- );
-
- // Jobs that just ran out of attempts are excluded from future runs by the
- // MAX_ATTEMPTS filter and need manual intervention
- const exhaustedJobs = jobs.filter(
- (job) =>
- failedResults.some((result) => result.id === job.id) &&
- job.attempts + 1 >= MAX_ATTEMPTS,
- );
-
- if (exhaustedJobs.length > 0) {
- for (const { id, name } of exhaustedJobs) {
- logger.error("jobs.retry_exhausted", { id, name });
- }
-
- await logger.flush();
- }
-}
diff --git a/apps/web/lib/jobs/publish-workflows.ts b/apps/web/lib/jobs/publish-workflows.ts
new file mode 100644
index 0000000000..ff45cc1922
--- /dev/null
+++ b/apps/web/lib/jobs/publish-workflows.ts
@@ -0,0 +1,385 @@
+import { createId } from "@/lib/api/create-id";
+import { logger } from "@/lib/axiom/server";
+import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
+import { Job, JobStatus, Prisma } from "@prisma/client";
+import { PublishRequest } from "@upstash/qstash";
+import { Client, TriggerOptions } from "@upstash/workflow";
+import { prisma } from "../prisma";
+import {
+ LAST_ERROR_MAX_LENGTH,
+ MAX_JOB_ATTEMPTS,
+ MAX_JOBS_PER_BATCH,
+ QSTASH_BATCH_CHUNK_SIZE,
+} from "./constants";
+
+const workflowClient = new Client({
+ baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",
+ token: process.env.QSTASH_TOKEN || "",
+ ...(process.env.VERCEL_ENV === "preview" && {
+ headers: {
+ "x-vercel-protection-bypass":
+ process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",
+ },
+ }),
+});
+
+const workflowPathMap = {
+ partnerApproved: "/api/workflows/partner-approved",
+ mergePartnerAccounts: "/api/workflows/merge-partner-accounts",
+ createPartnerCommission: "/api/workflows/create-partner-commission",
+} as const;
+
+export type WorkflowName = keyof typeof workflowPathMap;
+
+const WORKFLOW_NAMES = Object.keys(workflowPathMap) as WorkflowName[];
+
+type WorkflowOptions = Pick<
+ PublishRequest,
+ "label" | "deduplicationId" | "retries" | "flowControl"
+>;
+
+type DispatchWorkflowInput = {
+ name: WorkflowName;
+ payload: unknown;
+ options?: WorkflowOptions;
+};
+
+type WorkflowRecord = {
+ id: string;
+ name: WorkflowName;
+ payload: unknown;
+ options: WorkflowOptions;
+ createdAt?: Date;
+};
+
+type PublishResult = {
+ id: string;
+ status: typeof JobStatus.published | typeof JobStatus.failed;
+ lastError: string | null;
+ workflowRunId?: string;
+};
+
+export type DispatchWorkflowsResult = {
+ published: number;
+ failed: number;
+ results: PublishResult[];
+};
+
+export type PublishPendingWorkflowsResult = {
+ attempted: number;
+ published: number;
+ failed: number;
+};
+
+function toErrorFields(error: unknown) {
+ return {
+ errorName: error instanceof Error ? error.name : undefined,
+ errorMessage: error instanceof Error ? error.message : String(error),
+ };
+}
+
+function isTriggerSuccess(
+ response: unknown,
+): response is { workflowRunId: string } {
+ return (
+ typeof response === "object" &&
+ response !== null &&
+ "workflowRunId" in response &&
+ typeof response.workflowRunId === "string"
+ );
+}
+
+function buildTriggerRequest(job: WorkflowRecord): TriggerOptions {
+ const workflowPath = workflowPathMap[job.name];
+ const workflowKey = workflowPath.split("/").pop()!;
+
+ return {
+ url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,
+ body: job.payload,
+ workflowRunId: job.options.deduplicationId ?? job.id,
+ retries: job.options.retries ?? 5,
+ flowControl: job.options.flowControl ?? {
+ key: workflowKey,
+ parallelism: 15,
+ },
+ ...(job.options.label && { label: job.options.label }),
+ };
+}
+
+async function triggerWorkflows(
+ jobs: WorkflowRecord[],
+): Promise<PublishResult[]> {
+ if (jobs.length === 0) {
+ return [];
+ }
+
+ const results: PublishResult[] = [];
+
+ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
+ try {
+ const responses = await workflowClient.trigger(
+ jobChunk.map((job) => buildTriggerRequest(job)),
+ );
+
+ jobChunk.forEach((job, index) => {
+ const response = responses[index];
+
+ if (isTriggerSuccess(response)) {
+ results.push({
+ id: job.id,
+ status: JobStatus.published,
+ lastError: null,
+ workflowRunId: response.workflowRunId,
+ });
+ } else {
+ results.push({
+ id: job.id,
+ status: JobStatus.failed,
+ lastError: "Workflow trigger did not return a workflowRunId",
+ });
+ }
+ });
+ } catch (error) {
+ const lastError = serializeError(error).slice(0, LAST_ERROR_MAX_LENGTH);
+
+ logger.error("workflows.publish_failed", {
+ jobCount: jobChunk.length,
+ ...toErrorFields(error),
+ });
+
+ for (const job of jobChunk) {
+ results.push({
+ id: job.id,
+ status: JobStatus.failed,
+ lastError,
+ });
+ }
+ }
+ }
+
+ await logger.flush();
+
+ return results;
+}
+
+async function persistFailedWorkflows({
+ jobs,
+ failedResults,
+}: {
+ jobs: WorkflowRecord[];
+ failedResults: PublishResult[];
+}) {
+ if (failedResults.length === 0) {
+ return;
+ }
+
+ const failedById = new Map(
+ failedResults.map((result) => [result.id, result]),
+ );
+
+ const data = jobs
+ .filter((job) => failedById.has(job.id))
+ .map((job) => {
+ const failed = failedById.get(job.id)!;
+
+ return {
+ id: job.id,
+ name: job.name,
+ payload: job.payload as Prisma.InputJsonValue,
+ options: job.options as Prisma.InputJsonValue,
+ scheduledAt: new Date(),
+ status: JobStatus.failed,
+ lastError: failed.lastError,
+ attempts: 1,
+ };
+ });
+
+ try {
+ await prisma.job.createMany({
+ skipDuplicates: true,
+ data,
+ });
+ } catch (error) {
+ logger.error("workflows.dispatch_lost", {
+ jobCount: data.length,
+ ...toErrorFields(error),
+ });
+ await logger.flush();
+ }
+}
+
+async function deletePublishedWorkflows(publishedResults: PublishResult[]) {
+ if (publishedResults.length === 0) {
+ return;
+ }
+
+ await prisma.job.deleteMany({
+ where: {
+ id: {
+ in: publishedResults.map((result) => result.id),
+ },
+ },
+ });
+}
+
+async function markWorkflowsAsFailed({
+ failedResults,
+ jobs,
+}: {
+ failedResults: PublishResult[];
+ jobs: Pick<Job, "id" | "name" | "attempts">[];
+}) {
+ if (failedResults.length === 0) {
+ return;
+ }
+
+ const byError = new Map<string, string[]>();
+
+ for (const result of failedResults) {
+ const key = result.lastError ?? "";
+ const ids = byError.get(key) ?? [];
+ ids.push(result.id);
+ byError.set(key, ids);
+ }
+
+ await Promise.all(
+ [...byError.entries()].map(([lastError, ids]) =>
+ prisma.job.updateMany({
+ where: {
+ id: {
+ in: ids,
+ },
+ },
+ data: {
+ status: JobStatus.failed,
+ lastError,
+ attempts: {
+ increment: 1,
+ },
+ },
+ }),
+ ),
+ );
+
+ const exhaustedJobs = jobs.filter(
+ (job) =>
+ failedResults.some((result) => result.id === job.id) &&
+ job.attempts + 1 >= MAX_JOB_ATTEMPTS,
+ );
+
+ if (exhaustedJobs.length > 0) {
+ for (const { id, name } of exhaustedJobs) {
+ logger.error("workflows.retry_exhausted", { id, name });
+ }
+
+ await logger.flush();
+ }
+}
+
+/** Publish workflows to QStash; persist only on failure for the retry cron. */
+export async function dispatchWorkflows(
+ input: DispatchWorkflowInput | DispatchWorkflowInput[],
+): Promise<DispatchWorkflowsResult> {
+ const inputs = Array.isArray(input) ? input : [input];
+
+ if (inputs.length === 0) {
+ return {
+ published: 0,
+ failed: 0,
+ results: [],
+ };
+ }
+
+ const jobs: WorkflowRecord[] = inputs.map((job) => {
+ const id = createId({ prefix: "job_" });
+
+ return {
+ id,
+ name: job.name,
+ payload: job.payload,
+ // Default deduplicationId to the job id so inline publish and the
+ // retry cron cannot deliver the same workflow twice
+ options: {
+ ...job.options,
+ deduplicationId: job.options?.deduplicationId ?? id,
+ },
+ };
+ });
+
+ const results = await triggerWorkflows(jobs);
+
+ const publishedResults = results.filter(
+ (result) => result.status === JobStatus.published,
+ );
+ const failedResults = results.filter(
+ (result) => result.status === JobStatus.failed,
+ );
+
+ await persistFailedWorkflows({ jobs, failedResults });
+
+ return {
+ published: publishedResults.length,
+ failed: failedResults.length,
+ results,
+ };
+}
+
+/** Republish deferred workflows that failed to publish at dispatch time. */
+export async function publishPendingWorkflows(): Promise<PublishPendingWorkflowsResult> {
+ const emptyResult: PublishPendingWorkflowsResult = {
+ attempted: 0,
+ published: 0,
+ failed: 0,
+ };
+
+ const jobs = await prisma.job.findMany({
+ where: {
+ name: {
+ in: WORKFLOW_NAMES,
+ },
+ scheduledAt: {
+ lte: new Date(),
+ },
+ status: {
+ in: [JobStatus.pending, JobStatus.failed],
+ },
+ attempts: {
+ lt: MAX_JOB_ATTEMPTS,
+ },
+ },
+ orderBy: {
+ createdAt: "asc",
+ },
+ take: MAX_JOBS_PER_BATCH,
+ });
+
+ if (jobs.length === 0) {
+ return emptyResult;
+ }
+
+ const workflowJobs: WorkflowRecord[] = jobs.map((job) => ({
+ id: job.id,
+ name: job.name as WorkflowName,
+ payload: job.payload,
+ options: (job.options ?? {}) as WorkflowOptions,
+ createdAt: job.createdAt,
+ }));
+
+ const results = await triggerWorkflows(workflowJobs);
+
+ const publishedResults = results.filter(
+ (result) => result.status === JobStatus.published,
+ );
+ const failedResults = results.filter(
+ (result) => result.status === JobStatus.failed,
+ );
+
+ await deletePublishedWorkflows(publishedResults);
+ await markWorkflowsAsFailed({ failedResults, jobs });
+
+ return {
+ attempted: jobs.length,
+ published: publishedResults.length,
+ failed: failedResults.length,
+ };
+}
diff --git a/apps/web/lib/jobs/publishers/qstash-job-publisher.ts b/apps/web/lib/jobs/publishers/qstash-job-publisher.ts
deleted file mode 100644
index 0c4b8d3374..0000000000
--- a/apps/web/lib/jobs/publishers/qstash-job-publisher.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
-import { Job, JobStatus } from "@prisma/client";
-import { PublishBatchRequest } from "@upstash/qstash";
-import { qstash } from "../../cron";
-import { isPublishSuccess } from "../index";
-import { JobPublisher, PublishSendResult, QStashJobOptions } from "./types";
-
-const QSTASH_BATCH_CHUNK_SIZE = 100;
-
-const jobPathMap = {
- welcomeUser: "/api/cron/welcome-user",
-};
-
-export class QStashJobPublisher implements JobPublisher {
- async send(jobs: Job[]): Promise<PublishSendResult[]> {
- if (jobs.length === 0) {
- return [];
- }
-
- const results: PublishSendResult[] = [];
-
- for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
- try {
- const responses = await qstash.batchJSON(
- jobChunk.map((job) => this.buildRequest(job)),
- );
-
- jobChunk.forEach((job, index) => {
- if (isPublishSuccess(responses[index])) {
- results.push({
- id: job.id,
- status: JobStatus.published,
- lastError: null,
- });
- } else {
- results.push({
- id: job.id,
- status: JobStatus.failed,
- lastError: "QStash batch publish did not return a messageId",
- });
- }
- });
- } catch (error) {
- const lastError = serializeError(error);
-
- for (const job of jobChunk) {
- results.push({
- id: job.id,
- status: JobStatus.failed,
- lastError,
- });
- }
- }
- }
-
- return results;
- }
-
- private buildRequest(job: Job): PublishBatchRequest<unknown> {
- let jobPath = jobPathMap[job.name as keyof typeof jobPathMap];
-
- if (!jobPath) {
- jobPath = `/api/jobs/process/${job.name}`;
- }
-
- const options = (job.options ?? {}) as QStashJobOptions;
-
- return {
- url: `${APP_DOMAIN_WITH_NGROK}${jobPath}`,
- body: job.payload,
- deduplicationId: options.deduplicationId ?? job.id,
- ...(options.label && { label: options.label }),
- ...(options.retries !== undefined && { retries: options.retries }),
- ...(options.flowControl && { flowControl: options.flowControl }),
- ...(options.queue && { queueName: options.queue }),
- };
- }
-}
diff --git a/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts b/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts
deleted file mode 100644
index 3f83fea730..0000000000
--- a/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts
+++ /dev/null
@@ -1,106 +0,0 @@
-import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
-import { Job, JobStatus } from "@prisma/client";
-import { Client, TriggerOptions } from "@upstash/workflow";
-import { JobPublisher, PublishSendResult, QStashJobOptions } from "./types";
-
-const workflowClient = new Client({
- baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",
- token: process.env.QSTASH_TOKEN || "",
- ...(process.env.VERCEL_ENV === "preview" && {
- headers: {
- "x-vercel-protection-bypass":
- process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",
- },
- }),
-});
-
-const workflowPathMap = {
- partnerApproved: "/api/workflows/partner-approved",
- mergePartnerAccounts: "/api/workflows/merge-partner-accounts",
- createPartnerCommission: "/api/workflows/create-partner-commission",
-};
-
-const QSTASH_BATCH_CHUNK_SIZE = 100;
-
-type WorkflowJob = Job & { name: keyof typeof workflowPathMap };
-
-export class QStashWorkflowPublisher implements JobPublisher {
- static isWorkflowJob(job: Job): job is WorkflowJob {
- return job.name in workflowPathMap;
- }
-
- async send(jobs: Job[]): Promise<PublishSendResult[]> {
- const workflowJobs = jobs.filter(QStashWorkflowPublisher.isWorkflowJob);
-
- if (workflowJobs.length === 0) {
- return [];
- }
-
- const results: PublishSendResult[] = [];
-
- for (const jobChunk of chunk(workflowJobs, QSTASH_BATCH_CHUNK_SIZE)) {
- try {
- const responses = await workflowClient.trigger(
- jobChunk.map((job) => this.buildRequest(job)),
- );
-
- jobChunk.forEach((job, index) => {
- if (this.isTriggerSuccess(responses[index])) {
- results.push({
- id: job.id,
- status: JobStatus.published,
- lastError: null,
- });
- } else {
- results.push({
- id: job.id,
- status: JobStatus.failed,
- lastError: "Workflow trigger did not return a workflowRunId",
- });
- }
- });
- } catch (error) {
- const lastError = serializeError(error);
-
- for (const job of jobChunk) {
- results.push({
- id: job.id,
- status: JobStatus.failed,
- lastError,
- });
- }
- }
- }
-
- return results;
- }
-
- private buildRequest(job: WorkflowJob): TriggerOptions {
- const options = (job.options ?? {}) as QStashJobOptions;
- const workflowPath = workflowPathMap[job.name];
- const workflowKey = workflowPath.split("/").pop()!;
-
- return {
- url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,
- body: job.payload,
- workflowRunId: options.deduplicationId ?? job.id,
- retries: options.retries ?? 5,
- flowControl: options.flowControl ?? {
- key: workflowKey,
- parallelism: 15,
- },
- ...(options.label && { label: options.label }),
- };
- }
-
- private isTriggerSuccess(
- response: unknown,
- ): response is { workflowRunId: string } {
- return (
- typeof response === "object" &&
- response !== null &&
- "workflowRunId" in response &&
- typeof response.workflowRunId === "string"
- );
- }
-}
diff --git a/apps/web/lib/jobs/publishers/types.ts b/apps/web/lib/jobs/publishers/types.ts
deleted file mode 100644
index 7468d4de7b..0000000000
--- a/apps/web/lib/jobs/publishers/types.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import { Job, JobStatus } from "@prisma/client";
-import { PublishRequest } from "@upstash/qstash";
-
-export type QStashJobOptions = Pick<
- PublishRequest,
- "label" | "deduplicationId" | "retries" | "flowControl"
-> & {
- queue?: string;
-};
-
-export type PublishSendResult = {
- id: string;
- status: typeof JobStatus.published | typeof JobStatus.failed;
- lastError: string | null;
-};
-
-export interface JobPublisher {
- send(jobs: Job[]): Promise<PublishSendResult[]>;
-}
diff --git a/apps/web/lib/partners/queue-partner-commission-creation.ts b/apps/web/lib/partners/queue-partner-commission-creation.ts
index e1e5fff652..26578b09ba 100644
--- a/apps/web/lib/partners/queue-partner-commission-creation.ts
+++ b/apps/web/lib/partners/queue-partner-commission-creation.ts
@@ -1,5 +1,5 @@
import { getProgramEnrollmentOrThrow } from "../api/programs/get-program-enrollment-or-throw";
-import { dispatchJobs } from "../jobs/publish-jobs";
+import { dispatchWorkflows } from "../jobs/publish-workflows";
import { CreatePartnerCommissionProps } from "../types";
import { constructWebhookPartner } from "./constuct-webhook-partner";
@@ -19,7 +19,7 @@ export const queuePartnerCommissionCreation = async (
const { partner, links, ...programEnrollment } = result;
- await dispatchJobs({
+ await dispatchWorkflows({
name: "createPartnerCommission",
payload: params,
options: {
diff --git a/apps/web/vercel.json b/apps/web/vercel.json
index 7bbfb94172..e59dfb5eda 100644
--- a/apps/web/vercel.json
+++ b/apps/web/vercel.json
@@ -92,10 +92,6 @@
"path": "/api/cron/queue/retry",
"schedule": "* * * * *"
},
- {
- "path": "/api/cron/queue/cleanup",
- "schedule": "0 2 * * *"
- },
{
"path": "/api/cron/campaigns/queue-scheduled",
"schedule": "* * * * *"