The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Persist workflows only when QStash publish fails, and drop the write-ahead outbox.
Kiran K18dae57ff60f
diff --git a/apps/web/app/(ee)/api/cron/queue/cleanup/route.ts b/apps/web/app/(ee)/api/cron/queue/cleanup/route.tsdeleted file mode 100644index 921710da79..0000000000--- a/apps/web/app/(ee)/api/cron/queue/cleanup/route.ts+++ /dev/null@@ -1,21 +0,0 @@-import { withCron } from "@/lib/cron/with-cron";-import { cleanupPublishedJobs } from "@/lib/jobs/cleanup-jobs";-import { logAndRespond } from "../../utils";--export const dynamic = "force-dynamic";-export const maxDuration = 600;--// Deletes published Job rows older than 24 hours so the outbox does not grow unbounded.-// Runs once every day at 02:00:00 AM UTC (0 2 * * *)-// GET /api/cron/queue/cleanup-export const GET = withCron(async () => {- const { deletedCount } = await cleanupPublishedJobs();-- if (deletedCount === 0) {- return logAndRespond("No published jobs older than 24 hours to delete.");- }-- return logAndRespond(- `Deleted ${deletedCount} published jobs older than 24 hours.`,- );-});diff --git a/apps/web/app/(ee)/api/cron/queue/retry/route.ts b/apps/web/app/(ee)/api/cron/queue/retry/route.tsindex ccf2d2d520..f16bab8611 100644--- a/apps/web/app/(ee)/api/cron/queue/retry/route.ts+++ b/apps/web/app/(ee)/api/cron/queue/retry/route.ts@@ -1,5 +1,6 @@import { withCron } from "@/lib/cron/with-cron";-import { publishPendingJobs } from "@/lib/jobs/publish-jobs";+import { publishPendingDefineJobs } from "@/lib/jobs";+import { publishPendingWorkflows } from "@/lib/jobs/publish-workflows";import { redis } from "@/lib/upstash/redis";import { logAndRespond } from "../../utils";@@ -11,8 +12,8 @@ export const dynamic = "force-dynamic";const LOCK_KEY = "lock:queue-retry";const LOCK_TTL_SECONDS = 600;-// GET /api/cron/queue/retry – republish background jobs that failed to-// publish to QStash at dispatch time; rows are marked published on success+// GET /api/cron/queue/retry – republish workflows / defineJob rows that failed+// to publish to QStash at dispatch time; rows are deleted on successful publishexport const GET = withCron(async () => {const acquired = await redis.set(LOCK_KEY, "1", {nx: true,@@ -26,7 +27,14 @@ export const GET = withCron(async () => {}try {- const { attempted, published, failed } = await publishPendingJobs();+ const [workflows, defineJobs] = await Promise.all([+ publishPendingWorkflows(),+ publishPendingDefineJobs(),+ ]);++ const attempted = workflows.attempted + defineJobs.attempted;+ const published = workflows.published + defineJobs.published;+ const failed = workflows.failed + defineJobs.failed;if (attempted === 0) {return logAndRespond("No background jobs to retry.");diff --git a/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts b/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.tsindex d71b7b4330..1830d5ac7d 100644--- a/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts+++ b/apps/web/app/(ee)/api/e2e/trigger-merge-accounts/route.ts@@ -1,7 +1,7 @@import { DubApiError } from "@/lib/api/errors";import { parseRequestBody } from "@/lib/api/utils";import { withWorkspace } from "@/lib/auth";-import { dispatchJobs } from "@/lib/jobs/publish-jobs";+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";import { prisma } from "@/lib/prisma";import { ACME_PROGRAM_ID, nanoid } from "@dub/utils";import { NextResponse } from "next/server";@@ -52,7 +52,7 @@ export const POST = withWorkspace(const userId = `e2e-merge-${nanoid()}`;- const res = await dispatchJobs({+ const res = await dispatchWorkflows({name: "mergePartnerAccounts",payload: {userId,diff --git a/apps/web/app/(ee)/api/partner-profile/referrals/route.ts b/apps/web/app/(ee)/api/partner-profile/referrals/route.tsindex 832e1a264d..ccb2c7e673 100644--- a/apps/web/app/(ee)/api/partner-profile/referrals/route.ts+++ b/apps/web/app/(ee)/api/partner-profile/referrals/route.ts@@ -2,7 +2,7 @@ import { createId } from "@/lib/api/create-id";import { DubApiError } from "@/lib/api/errors";import { obfuscateCustomerEmail } from "@/lib/api/partner-profile/obfuscate-customer-email";import { withPartnerProfile } from "@/lib/auth/partner";-import { dispatchJobs } from "@/lib/jobs/publish-jobs";+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";import {getNetworkReferralsQuerySchema,networkReferralSchema,@@ -57,7 +57,7 @@ export const GET = withPartnerProfile(async ({ partner, searchParams }) => {"Program enrollment created in the last 1 min, most likely it's a new partner",);- await dispatchJobs({+ await dispatchWorkflows({name: "partnerApproved",payload: {programId: NETWORK_PROGRAM_ID,diff --git a/apps/web/lib/actions/partners/bulk-approve-partners.ts b/apps/web/lib/actions/partners/bulk-approve-partners.tsindex 822f2786ff..7a10b8b34f 100644--- a/apps/web/lib/actions/partners/bulk-approve-partners.ts+++ b/apps/web/lib/actions/partners/bulk-approve-partners.ts@@ -5,7 +5,7 @@ import { getGroupOrThrow } from "@/lib/api/groups/get-group-or-throw";import { queuePartnerSearchSync } from "@/lib/api/partners/queue-partner-search-sync";import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw";import { trackApplicationEvents } from "@/lib/application-events/update-application-event";-import { dispatchJobs } from "@/lib/jobs/publish-jobs";+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";import { throwIfPartnersLimitExceeded } from "@/lib/partners/throw-if-partners-limit-exceeded";import { prisma } from "@/lib/prisma";import { bulkApprovePartnersSchema } from "@/lib/zod/schemas/partners";@@ -143,7 +143,7 @@ export const bulkApprovePartnersAction = authActionClient})),),- dispatchJobs(+ dispatchWorkflows(updatedEnrollments.map(({ partnerId, programId }) => ({name: "partnerApproved",payload: {diff --git a/apps/web/lib/actions/partners/merge-partner-accounts.ts b/apps/web/lib/actions/partners/merge-partner-accounts.tsindex 51526b3f70..cfc9e7f3be 100644--- a/apps/web/lib/actions/partners/merge-partner-accounts.ts+++ b/apps/web/lib/actions/partners/merge-partner-accounts.ts@@ -1,7 +1,7 @@"use server";import { generateOTP } from "@/lib/auth/utils";-import { dispatchJobs } from "@/lib/jobs/publish-jobs";+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";import { prisma } from "@/lib/prisma";import { ratelimit, redis } from "@/lib/upstash";import { emailSchema } from "@/lib/zod/schemas/auth";@@ -340,7 +340,7 @@ const mergeAccounts = async ({ userId }: { userId: string }) => {const { sourceEmail, targetEmail } = accounts;- await dispatchJobs({+ await dispatchWorkflows({name: "mergePartnerAccounts",payload: {userId,diff --git a/apps/web/lib/api/partners/applications/approve-partner.ts b/apps/web/lib/api/partners/applications/approve-partner.tsindex ffd032ceb7..4b2097c120 100644--- a/apps/web/lib/api/partners/applications/approve-partner.ts+++ b/apps/web/lib/api/partners/applications/approve-partner.ts@@ -1,5 +1,5 @@import { trackApplicationEvents } from "@/lib/application-events/update-application-event";-import { dispatchJobs } from "@/lib/jobs/publish-jobs";+import { dispatchWorkflows } from "@/lib/jobs/publish-workflows";import { prisma } from "@/lib/prisma";import { ProgramEnrollmentStatus } from "@prisma/client";import { waitUntil } from "@vercel/functions";@@ -156,7 +156,7 @@ export async function approvePartner({partnerIds: [partnerId],}),- dispatchJobs({+ dispatchWorkflows({name: "partnerApproved",payload: {programId,diff --git a/apps/web/lib/jobs/cleanup-jobs.ts b/apps/web/lib/jobs/cleanup-jobs.tsdeleted file mode 100644index 2171da8e89..0000000000--- a/apps/web/lib/jobs/cleanup-jobs.ts+++ /dev/null@@ -1,52 +0,0 @@-import { prisma } from "@/lib/prisma";-import { JobStatus } from "@prisma/client";-import { subHours } from "date-fns";--const MAX_JOBS_PER_BATCH = 1000;-const RETENTION_HOURS = 24;--export async function cleanupPublishedJobs(): Promise<{- deletedCount: number;-}> {- const cutoff = subHours(new Date(), RETENTION_HOURS);- let deletedCount = 0;-- while (true) {- const jobs = await prisma.job.findMany({- where: {- status: JobStatus.published,- createdAt: {- lt: cutoff,- },- },- select: {- id: true,- },- take: MAX_JOBS_PER_BATCH,- });-- if (jobs.length === 0) {- break;- }-- const { count } = await prisma.job.deleteMany({- where: {- id: {- in: jobs.map((job) => job.id),- },- },- });-- deletedCount += count;-- if (jobs.length < MAX_JOBS_PER_BATCH) {- break;- }-- await new Promise((resolve) => setTimeout(resolve, 500));- }-- return {- deletedCount,- };-}diff --git a/apps/web/lib/jobs/constants.ts b/apps/web/lib/jobs/constants.tsnew file mode 100644index 0000000000..0d73eb9cc5--- /dev/null+++ b/apps/web/lib/jobs/constants.ts@@ -0,0 +1,4 @@+export const QSTASH_BATCH_CHUNK_SIZE = 100;+export const LAST_ERROR_MAX_LENGTH = 1000;+export const MAX_JOB_ATTEMPTS = 10;+export const MAX_JOBS_PER_BATCH = 100;diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.tsindex 237a41d751..1d09986b5e 100644--- a/apps/web/lib/jobs/index.ts+++ b/apps/web/lib/jobs/index.ts@@ -3,9 +3,15 @@ import { logger } from "@/lib/axiom/server";import { qstash } from "@/lib/cron";import { prisma } from "@/lib/prisma";import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";-import { Prisma } from "@prisma/client";+import { JobStatus, Prisma } from "@prisma/client";import { PublishRequest } from "@upstash/qstash";import * as z from "zod/v4";+import {+ LAST_ERROR_MAX_LENGTH,+ MAX_JOB_ATTEMPTS,+ MAX_JOBS_PER_BATCH,+ QSTASH_BATCH_CHUNK_SIZE,+} from "./constants";type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;@@ -50,8 +56,6 @@ const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;const QSTASH_PUBLISH_MAX_RETRIES = 3;-const QSTASH_BATCH_CHUNK_SIZE = 100;-export const jobNameSchema = z.string().regex(@@ -110,18 +114,26 @@ async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {throw new Error("Failed to publish to QStash.");}+type JobReplayOptions = Pick<+ JobDispatchOptions,+ "deduplicationId" | "retries" | "queue" | "flowControl" | "label"+>;+function buildQStashJobRequest({ name, payload, options }: DispatchJobInput,opts?: {+ dispatchedAt?: string;batch?: boolean;+ notBefore?: number;},) {const envelope: JobEnvelope = {name,payload,- dispatchedAt: new Date().toISOString(),+ dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),};+ const notBefore = opts?.notBefore ?? options?.notBefore;const deduplicationId = buildJobDeduplicationId(name,options?.deduplicationId,@@ -131,8 +143,11 @@ function buildQStashJobRequest(url: getJobsEndpointUrl(name),body: envelope,label: buildJobLabel(name, options?.label),- ...(options?.delay && { delay: options.delay }),- ...(options?.notBefore && { notBefore: options.notBefore }),+ ...(options?.delay &&+ opts?.notBefore === undefined && {+ delay: options.delay,+ }),+ ...(notBefore && { notBefore }),...(deduplicationId && { deduplicationId }),...(options?.retries !== undefined && { retries: options.retries }),...(options?.flowControl && { flowControl: options.flowControl }),@@ -140,6 +155,38 @@ function buildQStashJobRequest(};}+// Used by /api/cron/queue/retry to republish deferred defineJob rows+export function buildReplayRequest(+ job: {+ name: string;+ payload: Prisma.JsonValue;+ options: Prisma.JsonValue | null;+ createdAt: Date;+ scheduledAt: Date | null;+ },+ now: Date = new Date(),+) {+ const options = (job.options ?? {}) as JobReplayOptions;++ const notBefore =+ job.scheduledAt && job.scheduledAt > now+ ? Math.floor(job.scheduledAt.getTime() / 1000)+ : undefined;++ return buildQStashJobRequest(+ {+ name: job.name,+ payload: job.payload,+ options,+ },+ {+ dispatchedAt: job.createdAt.toISOString(),+ batch: true,+ notBefore,+ },+ );+}+export function isPublishSuccess(response: unknown,): response is { messageId: string } {@@ -401,3 +448,113 @@ export function defineJob<TSchema extends z.ZodType>({}export type JobDefinition = ReturnType<typeof defineJob<z.ZodType>>;++/** Republish deferred defineJob rows that failed to publish at dispatch time. */+export async function publishPendingDefineJobs(): Promise<{+ attempted: number;+ published: number;+ failed: number;+}> {+ const jobs = await prisma.job.findMany({+ where: {+ name: {+ endsWith: "-job",+ },+ scheduledAt: {+ lte: new Date(),+ },+ status: {+ in: [JobStatus.pending, JobStatus.failed],+ },+ attempts: {+ lt: MAX_JOB_ATTEMPTS,+ },+ },+ orderBy: {+ createdAt: "asc",+ },+ take: MAX_JOBS_PER_BATCH,+ });++ if (jobs.length === 0) {+ return { attempted: 0, published: 0, failed: 0 };+ }++ const now = new Date();+ const publishedIds: string[] = [];+ const failedByError = new Map<string, string[]>();++ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {+ try {+ const responses = await qstash.batchJSON(+ jobChunk.map((job) => buildReplayRequest(job, now)),+ );++ jobChunk.forEach((job, index) => {+ if (isPublishSuccess(responses[index])) {+ publishedIds.push(job.id);+ } else {+ const key = "QStash batch publish did not return a messageId";+ const ids = failedByError.get(key) ?? [];+ ids.push(job.id);+ failedByError.set(key, ids);+ }+ });+ } catch (error) {+ const lastError = (+ error instanceof Error ? error.message : String(error)+ ).slice(0, LAST_ERROR_MAX_LENGTH);++ for (const job of jobChunk) {+ const ids = failedByError.get(lastError) ?? [];+ ids.push(job.id);+ failedByError.set(lastError, ids);+ }+ }+ }++ if (publishedIds.length > 0) {+ await prisma.job.deleteMany({+ where: {+ id: {+ in: publishedIds,+ },+ },+ });+ }++ for (const [lastError, ids] of failedByError) {+ await prisma.job.updateMany({+ where: {+ id: {+ in: ids,+ },+ },+ data: {+ status: JobStatus.failed,+ lastError,+ attempts: {+ increment: 1,+ },+ },+ });+ }++ const failedIds = new Set([...failedByError.values()].flat());+ const exhaustedJobs = jobs.filter(+ (job) => failedIds.has(job.id) && job.attempts + 1 >= MAX_JOB_ATTEMPTS,+ );++ if (exhaustedJobs.length > 0) {+ logger.error("jobs.retry_exhausted", {+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),+ });+ await logger.flush();+ }++ return {+ attempted: jobs.length,+ published: publishedIds.length,+ failed: failedIds.size,+ };+}diff --git a/apps/web/lib/jobs/publish-jobs.ts b/apps/web/lib/jobs/publish-jobs.tsdeleted file mode 100644index eb4e842301..0000000000--- a/apps/web/lib/jobs/publish-jobs.ts+++ /dev/null@@ -1,204 +0,0 @@-import { logger } from "@/lib/axiom/server";-import { pluck } from "@dub/utils";-import { Job, JobStatus, Prisma } from "@prisma/client";-import { createId } from "../api/create-id";-import { prisma } from "../prisma";-import { QStashJobPublisher } from "./publishers/qstash-job-publisher";-import { QStashWorkflowPublisher } from "./publishers/qstash-workflow-publisher";-import type { PublishSendResult, QStashJobOptions } from "./publishers/types";--const MAX_ATTEMPTS = 10;-const MAX_JOBS_PER_BATCH = 100;--type DispatchJobInput = {- name: string;- payload: unknown;- options?: QStashJobOptions;-};--export type PublishPendingJobsResult = {- attempted: number;- published: number;- failed: number;-};--const qstashJobPublisher = new QStashJobPublisher();-const qstashWorkflowPublisher = new QStashWorkflowPublisher();--export async function dispatchJobs(- input: DispatchJobInput | DispatchJobInput[],-) {- const jobs = Array.isArray(input) ? input : [input];-- if (jobs.length === 0) {- return;- }-- const data = jobs.map((job) => {- const id = createId({ prefix: "job_" });-- return {- id,- name: job.name,- scheduledAt: new Date(),- payload: job.payload as Prisma.InputJsonValue,- // Default deduplicationId to the job id so best-effort publish and the- // retry cron cannot deliver the same job twice via QStash- options: {- ...job.options,- deduplicationId: job.options?.deduplicationId ?? id,- } as Prisma.InputJsonValue,- };- });-- // Durably persist the jobs in the database- await prisma.job.createMany({- skipDuplicates: true,- data: data,- });-- const jobIds = pluck(data, "id");-- // Best effort publish, if the job is not published, it will be published on the next cron run- await publishPendingJobs(jobIds);-}--export async function publishPendingJobs(- jobIds?: string[],-): Promise<PublishPendingJobsResult> {- const emptyResult: PublishPendingJobsResult = {- attempted: 0,- published: 0,- failed: 0,- };-- if (jobIds && jobIds.length === 0) {- return emptyResult;- }-- const isScanMode = jobIds === undefined;-- const jobs = await prisma.job.findMany({- where: {- ...(jobIds && {- id: {- in: jobIds,- },- }),- scheduledAt: {- lte: new Date(),- },- status: {- in: [JobStatus.pending, JobStatus.failed],- },- attempts: {- lt: MAX_ATTEMPTS,- },- },- ...(isScanMode && {- orderBy: {- createdAt: "asc" as const,- },- take: MAX_JOBS_PER_BATCH,- }),- });-- if (jobs.length === 0) {- return emptyResult;- }-- const workflowJobs = jobs.filter(QStashWorkflowPublisher.isWorkflowJob);- const qstashJobs = jobs.filter(- (job) => !QStashWorkflowPublisher.isWorkflowJob(job),- );-- const results = (- await Promise.all([- qstashWorkflowPublisher.send(workflowJobs),- qstashJobPublisher.send(qstashJobs),- ])- ).flat();-- const publishedResults = results.filter(- (result) => result.status === JobStatus.published,- );-- const failedResults = results.filter(- (result) => result.status === JobStatus.failed,- );-- await markAsPublished({ publishedResults });- await markAsFailed({ failedResults, jobs });-- return {- attempted: jobs.length,- published: publishedResults.length,- failed: failedResults.length,- };-}--async function markAsPublished({- publishedResults,-}: {- publishedResults: PublishSendResult[];-}) {- if (publishedResults.length === 0) {- return;- }-- await prisma.job.updateMany({- where: {- id: {- in: publishedResults.map((result) => result.id),- },- },- data: {- status: JobStatus.published,- },- });-}--async function markAsFailed({- failedResults,- jobs,-}: {- failedResults: PublishSendResult[];- jobs: Pick<Job, "id" | "name" | "attempts">[];-}) {- if (failedResults.length === 0) {- return;- }-- await Promise.all(- failedResults.map((result) =>- prisma.job.update({- where: {- id: result.id,- },- data: {- status: JobStatus.failed,- lastError: result.lastError,- attempts: {- increment: 1,- },- },- }),- ),- );-- // Jobs that just ran out of attempts are excluded from future runs by the- // MAX_ATTEMPTS filter and need manual intervention- const exhaustedJobs = jobs.filter(- (job) =>- failedResults.some((result) => result.id === job.id) &&- job.attempts + 1 >= MAX_ATTEMPTS,- );-- if (exhaustedJobs.length > 0) {- for (const { id, name } of exhaustedJobs) {- logger.error("jobs.retry_exhausted", { id, name });- }-- await logger.flush();- }-}diff --git a/apps/web/lib/jobs/publish-workflows.ts b/apps/web/lib/jobs/publish-workflows.tsnew file mode 100644index 0000000000..ff45cc1922--- /dev/null+++ b/apps/web/lib/jobs/publish-workflows.ts@@ -0,0 +1,385 @@+import { createId } from "@/lib/api/create-id";+import { logger } from "@/lib/axiom/server";+import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";+import { Job, JobStatus, Prisma } from "@prisma/client";+import { PublishRequest } from "@upstash/qstash";+import { Client, TriggerOptions } from "@upstash/workflow";+import { prisma } from "../prisma";+import {+ LAST_ERROR_MAX_LENGTH,+ MAX_JOB_ATTEMPTS,+ MAX_JOBS_PER_BATCH,+ QSTASH_BATCH_CHUNK_SIZE,+} from "./constants";++const workflowClient = new Client({+ baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",+ token: process.env.QSTASH_TOKEN || "",+ ...(process.env.VERCEL_ENV === "preview" && {+ headers: {+ "x-vercel-protection-bypass":+ process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",+ },+ }),+});++const workflowPathMap = {+ partnerApproved: "/api/workflows/partner-approved",+ mergePartnerAccounts: "/api/workflows/merge-partner-accounts",+ createPartnerCommission: "/api/workflows/create-partner-commission",+} as const;++export type WorkflowName = keyof typeof workflowPathMap;++const WORKFLOW_NAMES = Object.keys(workflowPathMap) as WorkflowName[];++type WorkflowOptions = Pick<+ PublishRequest,+ "label" | "deduplicationId" | "retries" | "flowControl"+>;++type DispatchWorkflowInput = {+ name: WorkflowName;+ payload: unknown;+ options?: WorkflowOptions;+};++type WorkflowRecord = {+ id: string;+ name: WorkflowName;+ payload: unknown;+ options: WorkflowOptions;+ createdAt?: Date;+};++type PublishResult = {+ id: string;+ status: typeof JobStatus.published | typeof JobStatus.failed;+ lastError: string | null;+ workflowRunId?: string;+};++export type DispatchWorkflowsResult = {+ published: number;+ failed: number;+ results: PublishResult[];+};++export type PublishPendingWorkflowsResult = {+ attempted: number;+ published: number;+ failed: number;+};++function toErrorFields(error: unknown) {+ return {+ errorName: error instanceof Error ? error.name : undefined,+ errorMessage: error instanceof Error ? error.message : String(error),+ };+}++function isTriggerSuccess(+ response: unknown,+): response is { workflowRunId: string } {+ return (+ typeof response === "object" &&+ response !== null &&+ "workflowRunId" in response &&+ typeof response.workflowRunId === "string"+ );+}++function buildTriggerRequest(job: WorkflowRecord): TriggerOptions {+ const workflowPath = workflowPathMap[job.name];+ const workflowKey = workflowPath.split("/").pop()!;++ return {+ url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,+ body: job.payload,+ workflowRunId: job.options.deduplicationId ?? job.id,+ retries: job.options.retries ?? 5,+ flowControl: job.options.flowControl ?? {+ key: workflowKey,+ parallelism: 15,+ },+ ...(job.options.label && { label: job.options.label }),+ };+}++async function triggerWorkflows(+ jobs: WorkflowRecord[],+): Promise<PublishResult[]> {+ if (jobs.length === 0) {+ return [];+ }++ const results: PublishResult[] = [];++ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {+ try {+ const responses = await workflowClient.trigger(+ jobChunk.map((job) => buildTriggerRequest(job)),+ );++ jobChunk.forEach((job, index) => {+ const response = responses[index];++ if (isTriggerSuccess(response)) {+ results.push({+ id: job.id,+ status: JobStatus.published,+ lastError: null,+ workflowRunId: response.workflowRunId,+ });+ } else {+ results.push({+ id: job.id,+ status: JobStatus.failed,+ lastError: "Workflow trigger did not return a workflowRunId",+ });+ }+ });+ } catch (error) {+ const lastError = serializeError(error).slice(0, LAST_ERROR_MAX_LENGTH);++ logger.error("workflows.publish_failed", {+ jobCount: jobChunk.length,+ ...toErrorFields(error),+ });++ for (const job of jobChunk) {+ results.push({+ id: job.id,+ status: JobStatus.failed,+ lastError,+ });+ }+ }+ }++ await logger.flush();++ return results;+}++async function persistFailedWorkflows({+ jobs,+ failedResults,+}: {+ jobs: WorkflowRecord[];+ failedResults: PublishResult[];+}) {+ if (failedResults.length === 0) {+ return;+ }++ const failedById = new Map(+ failedResults.map((result) => [result.id, result]),+ );++ const data = jobs+ .filter((job) => failedById.has(job.id))+ .map((job) => {+ const failed = failedById.get(job.id)!;++ return {+ id: job.id,+ name: job.name,+ payload: job.payload as Prisma.InputJsonValue,+ options: job.options as Prisma.InputJsonValue,+ scheduledAt: new Date(),+ status: JobStatus.failed,+ lastError: failed.lastError,+ attempts: 1,+ };+ });++ try {+ await prisma.job.createMany({+ skipDuplicates: true,+ data,+ });+ } catch (error) {+ logger.error("workflows.dispatch_lost", {+ jobCount: data.length,+ ...toErrorFields(error),+ });+ await logger.flush();+ }+}++async function deletePublishedWorkflows(publishedResults: PublishResult[]) {+ if (publishedResults.length === 0) {+ return;+ }++ await prisma.job.deleteMany({+ where: {+ id: {+ in: publishedResults.map((result) => result.id),+ },+ },+ });+}++async function markWorkflowsAsFailed({+ failedResults,+ jobs,+}: {+ failedResults: PublishResult[];+ jobs: Pick<Job, "id" | "name" | "attempts">[];+}) {+ if (failedResults.length === 0) {+ return;+ }++ const byError = new Map<string, string[]>();++ for (const result of failedResults) {+ const key = result.lastError ?? "";+ const ids = byError.get(key) ?? [];+ ids.push(result.id);+ byError.set(key, ids);+ }++ await Promise.all(+ [...byError.entries()].map(([lastError, ids]) =>+ prisma.job.updateMany({+ where: {+ id: {+ in: ids,+ },+ },+ data: {+ status: JobStatus.failed,+ lastError,+ attempts: {+ increment: 1,+ },+ },+ }),+ ),+ );++ const exhaustedJobs = jobs.filter(+ (job) =>+ failedResults.some((result) => result.id === job.id) &&+ job.attempts + 1 >= MAX_JOB_ATTEMPTS,+ );++ if (exhaustedJobs.length > 0) {+ for (const { id, name } of exhaustedJobs) {+ logger.error("workflows.retry_exhausted", { id, name });+ }++ await logger.flush();+ }+}++/** Publish workflows to QStash; persist only on failure for the retry cron. */+export async function dispatchWorkflows(+ input: DispatchWorkflowInput | DispatchWorkflowInput[],+): Promise<DispatchWorkflowsResult> {+ const inputs = Array.isArray(input) ? input : [input];++ if (inputs.length === 0) {+ return {+ published: 0,+ failed: 0,+ results: [],+ };+ }++ const jobs: WorkflowRecord[] = inputs.map((job) => {+ const id = createId({ prefix: "job_" });++ return {+ id,+ name: job.name,+ payload: job.payload,+ // Default deduplicationId to the job id so inline publish and the+ // retry cron cannot deliver the same workflow twice+ options: {+ ...job.options,+ deduplicationId: job.options?.deduplicationId ?? id,+ },+ };+ });++ const results = await triggerWorkflows(jobs);++ const publishedResults = results.filter(+ (result) => result.status === JobStatus.published,+ );+ const failedResults = results.filter(+ (result) => result.status === JobStatus.failed,+ );++ await persistFailedWorkflows({ jobs, failedResults });++ return {+ published: publishedResults.length,+ failed: failedResults.length,+ results,+ };+}++/** Republish deferred workflows that failed to publish at dispatch time. */+export async function publishPendingWorkflows(): Promise<PublishPendingWorkflowsResult> {+ const emptyResult: PublishPendingWorkflowsResult = {+ attempted: 0,+ published: 0,+ failed: 0,+ };++ const jobs = await prisma.job.findMany({+ where: {+ name: {+ in: WORKFLOW_NAMES,+ },+ scheduledAt: {+ lte: new Date(),+ },+ status: {+ in: [JobStatus.pending, JobStatus.failed],+ },+ attempts: {+ lt: MAX_JOB_ATTEMPTS,+ },+ },+ orderBy: {+ createdAt: "asc",+ },+ take: MAX_JOBS_PER_BATCH,+ });++ if (jobs.length === 0) {+ return emptyResult;+ }++ const workflowJobs: WorkflowRecord[] = jobs.map((job) => ({+ id: job.id,+ name: job.name as WorkflowName,+ payload: job.payload,+ options: (job.options ?? {}) as WorkflowOptions,+ createdAt: job.createdAt,+ }));++ const results = await triggerWorkflows(workflowJobs);++ const publishedResults = results.filter(+ (result) => result.status === JobStatus.published,+ );+ const failedResults = results.filter(+ (result) => result.status === JobStatus.failed,+ );++ await deletePublishedWorkflows(publishedResults);+ await markWorkflowsAsFailed({ failedResults, jobs });++ return {+ attempted: jobs.length,+ published: publishedResults.length,+ failed: failedResults.length,+ };+}diff --git a/apps/web/lib/jobs/publishers/qstash-job-publisher.ts b/apps/web/lib/jobs/publishers/qstash-job-publisher.tsdeleted file mode 100644index 0c4b8d3374..0000000000--- a/apps/web/lib/jobs/publishers/qstash-job-publisher.ts+++ /dev/null@@ -1,78 +0,0 @@-import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";-import { Job, JobStatus } from "@prisma/client";-import { PublishBatchRequest } from "@upstash/qstash";-import { qstash } from "../../cron";-import { isPublishSuccess } from "../index";-import { JobPublisher, PublishSendResult, QStashJobOptions } from "./types";--const QSTASH_BATCH_CHUNK_SIZE = 100;--const jobPathMap = {- welcomeUser: "/api/cron/welcome-user",-};--export class QStashJobPublisher implements JobPublisher {- async send(jobs: Job[]): Promise<PublishSendResult[]> {- if (jobs.length === 0) {- return [];- }-- const results: PublishSendResult[] = [];-- for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {- try {- const responses = await qstash.batchJSON(- jobChunk.map((job) => this.buildRequest(job)),- );-- jobChunk.forEach((job, index) => {- if (isPublishSuccess(responses[index])) {- results.push({- id: job.id,- status: JobStatus.published,- lastError: null,- });- } else {- results.push({- id: job.id,- status: JobStatus.failed,- lastError: "QStash batch publish did not return a messageId",- });- }- });- } catch (error) {- const lastError = serializeError(error);-- for (const job of jobChunk) {- results.push({- id: job.id,- status: JobStatus.failed,- lastError,- });- }- }- }-- return results;- }-- private buildRequest(job: Job): PublishBatchRequest<unknown> {- let jobPath = jobPathMap[job.name as keyof typeof jobPathMap];-- if (!jobPath) {- jobPath = `/api/jobs/process/${job.name}`;- }-- const options = (job.options ?? {}) as QStashJobOptions;-- return {- url: `${APP_DOMAIN_WITH_NGROK}${jobPath}`,- body: job.payload,- deduplicationId: options.deduplicationId ?? job.id,- ...(options.label && { label: options.label }),- ...(options.retries !== undefined && { retries: options.retries }),- ...(options.flowControl && { flowControl: options.flowControl }),- ...(options.queue && { queueName: options.queue }),- };- }-}diff --git a/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts b/apps/web/lib/jobs/publishers/qstash-workflow-publisher.tsdeleted file mode 100644index 3f83fea730..0000000000--- a/apps/web/lib/jobs/publishers/qstash-workflow-publisher.ts+++ /dev/null@@ -1,106 +0,0 @@-import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";-import { Job, JobStatus } from "@prisma/client";-import { Client, TriggerOptions } from "@upstash/workflow";-import { JobPublisher, PublishSendResult, QStashJobOptions } from "./types";--const workflowClient = new Client({- baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",- token: process.env.QSTASH_TOKEN || "",- ...(process.env.VERCEL_ENV === "preview" && {- headers: {- "x-vercel-protection-bypass":- process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",- },- }),-});--const workflowPathMap = {- partnerApproved: "/api/workflows/partner-approved",- mergePartnerAccounts: "/api/workflows/merge-partner-accounts",- createPartnerCommission: "/api/workflows/create-partner-commission",-};--const QSTASH_BATCH_CHUNK_SIZE = 100;--type WorkflowJob = Job & { name: keyof typeof workflowPathMap };--export class QStashWorkflowPublisher implements JobPublisher {- static isWorkflowJob(job: Job): job is WorkflowJob {- return job.name in workflowPathMap;- }-- async send(jobs: Job[]): Promise<PublishSendResult[]> {- const workflowJobs = jobs.filter(QStashWorkflowPublisher.isWorkflowJob);-- if (workflowJobs.length === 0) {- return [];- }-- const results: PublishSendResult[] = [];-- for (const jobChunk of chunk(workflowJobs, QSTASH_BATCH_CHUNK_SIZE)) {- try {- const responses = await workflowClient.trigger(- jobChunk.map((job) => this.buildRequest(job)),- );-- jobChunk.forEach((job, index) => {- if (this.isTriggerSuccess(responses[index])) {- results.push({- id: job.id,- status: JobStatus.published,- lastError: null,- });- } else {- results.push({- id: job.id,- status: JobStatus.failed,- lastError: "Workflow trigger did not return a workflowRunId",- });- }- });- } catch (error) {- const lastError = serializeError(error);-- for (const job of jobChunk) {- results.push({- id: job.id,- status: JobStatus.failed,- lastError,- });- }- }- }-- return results;- }-- private buildRequest(job: WorkflowJob): TriggerOptions {- const options = (job.options ?? {}) as QStashJobOptions;- const workflowPath = workflowPathMap[job.name];- const workflowKey = workflowPath.split("/").pop()!;-- return {- url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,- body: job.payload,- workflowRunId: options.deduplicationId ?? job.id,- retries: options.retries ?? 5,- flowControl: options.flowControl ?? {- key: workflowKey,- parallelism: 15,- },- ...(options.label && { label: options.label }),- };- }-- private isTriggerSuccess(- response: unknown,- ): response is { workflowRunId: string } {- return (- typeof response === "object" &&- response !== null &&- "workflowRunId" in response &&- typeof response.workflowRunId === "string"- );- }-}diff --git a/apps/web/lib/jobs/publishers/types.ts b/apps/web/lib/jobs/publishers/types.tsdeleted file mode 100644index 7468d4de7b..0000000000--- a/apps/web/lib/jobs/publishers/types.ts+++ /dev/null@@ -1,19 +0,0 @@-import { Job, JobStatus } from "@prisma/client";-import { PublishRequest } from "@upstash/qstash";--export type QStashJobOptions = Pick<- PublishRequest,- "label" | "deduplicationId" | "retries" | "flowControl"-> & {- queue?: string;-};--export type PublishSendResult = {- id: string;- status: typeof JobStatus.published | typeof JobStatus.failed;- lastError: string | null;-};--export interface JobPublisher {- send(jobs: Job[]): Promise<PublishSendResult[]>;-}diff --git a/apps/web/lib/partners/queue-partner-commission-creation.ts b/apps/web/lib/partners/queue-partner-commission-creation.tsindex e1e5fff652..26578b09ba 100644--- a/apps/web/lib/partners/queue-partner-commission-creation.ts+++ b/apps/web/lib/partners/queue-partner-commission-creation.ts@@ -1,5 +1,5 @@import { getProgramEnrollmentOrThrow } from "../api/programs/get-program-enrollment-or-throw";-import { dispatchJobs } from "../jobs/publish-jobs";+import { dispatchWorkflows } from "../jobs/publish-workflows";import { CreatePartnerCommissionProps } from "../types";import { constructWebhookPartner } from "./constuct-webhook-partner";@@ -19,7 +19,7 @@ export const queuePartnerCommissionCreation = async (const { partner, links, ...programEnrollment } = result;- await dispatchJobs({+ await dispatchWorkflows({name: "createPartnerCommission",payload: params,options: {diff --git a/apps/web/vercel.json b/apps/web/vercel.jsonindex 7bbfb94172..e59dfb5eda 100644--- a/apps/web/vercel.json+++ b/apps/web/vercel.json@@ -92,10 +92,6 @@"path": "/api/cron/queue/retry","schedule": "* * * * *"},- {- "path": "/api/cron/queue/cleanup",- "schedule": "0 2 * * *"- },{"path": "/api/cron/campaigns/queue-scheduled","schedule": "* * * * *"