The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Split job publishing into shared outbox transports and drop Job status.
Kiran K383f7b4433c6
diff --git a/apps/web/app/(ee)/api/cron/queue/retry/route.ts b/apps/web/app/(ee)/api/cron/queue/retry/route.tsindex f16bab8611..f0f09985b4 100644--- a/apps/web/app/(ee)/api/cron/queue/retry/route.ts+++ b/apps/web/app/(ee)/api/cron/queue/retry/route.ts@@ -1,6 +1,5 @@import { withCron } from "@/lib/cron/with-cron";-import { publishPendingDefineJobs } from "@/lib/jobs";-import { publishPendingWorkflows } from "@/lib/jobs/publish-workflows";+import { publishPendingJobs } from "@/lib/jobs/outbox";import { redis } from "@/lib/upstash/redis";import { logAndRespond } from "../../utils";@@ -12,8 +11,8 @@ export const dynamic = "force-dynamic";const LOCK_KEY = "lock:queue-retry";const LOCK_TTL_SECONDS = 600;-// GET /api/cron/queue/retry – republish workflows / defineJob rows that failed-// to publish to QStash at dispatch time; rows are deleted on successful publish+// GET /api/cron/queue/retry – republish jobs that failed to publish to QStash+// at dispatch time; rows are deleted on successful publishexport const GET = withCron(async () => {const acquired = await redis.set(LOCK_KEY, "1", {nx: true,@@ -27,14 +26,7 @@ export const GET = withCron(async () => {}try {- const [workflows, defineJobs] = await Promise.all([- publishPendingWorkflows(),- publishPendingDefineJobs(),- ]);-- const attempted = workflows.attempted + defineJobs.attempted;- const published = workflows.published + defineJobs.published;- const failed = workflows.failed + defineJobs.failed;+ const { attempted, published, failed } = await publishPendingJobs();if (attempted === 0) {return logAndRespond("No background jobs to retry.");diff --git a/apps/web/app/api/jobs/process/[jobName]/route.ts b/apps/web/app/api/jobs/process/[jobName]/route.tsindex 57c5001983..3b0e2f46c2 100644--- a/apps/web/app/api/jobs/process/[jobName]/route.ts+++ b/apps/web/app/api/jobs/process/[jobName]/route.ts@@ -1,7 +1,7 @@import { handleAndReturnErrorResponse } from "@/lib/api/errors";import { logger, withAxiomBodyLog } from "@/lib/axiom/server";import { verifyQstashSignature } from "@/lib/cron/verify-qstash";-import { jobEnvelopeSchema } from "@/lib/jobs";+import { jobEnvelopeSchema } from "@/lib/jobs/send-jobs";import { loadJob } from "@/lib/jobs/registry";import * as z from "zod/v4";diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.tsindex 1d09986b5e..94650c516b 100644--- a/apps/web/lib/jobs/index.ts+++ b/apps/web/lib/jobs/index.ts@@ -1,33 +1,18 @@import { createId } from "@/lib/api/create-id";-import { logger } from "@/lib/axiom/server";+import { logger, toErrorFields } from "@/lib/axiom/server";import { qstash } from "@/lib/cron";import { prisma } from "@/lib/prisma";-import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";-import { JobStatus, Prisma } from "@prisma/client";-import { PublishRequest } from "@upstash/qstash";+import { chunk } from "@dub/utils";+import { Prisma } from "@prisma/client";import * as z from "zod/v4";+import { QSTASH_BATCH_CHUNK_SIZE } from "./constants";import {- LAST_ERROR_MAX_LENGTH,- MAX_JOB_ATTEMPTS,- MAX_JOBS_PER_BATCH,- QSTASH_BATCH_CHUNK_SIZE,-} from "./constants";--type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;--type JobPublishOptions = Pick<- PublishRequest,- | "delay"- | "notBefore"- | "deduplicationId"- | "retries"- | "flowControl"- | "label"->;--export type JobDispatchOptions = JobPublishOptions & {- queue?: string;-};+ buildJobLabel,+ buildQStashJobRequest,+ isPublishSuccess,+ jobNameSchema,+ type JobDispatchOptions,+} from "./send-jobs";// Per-job defaults, merged under per-dispatch optionsexport type JobDefaults = Pick<@@ -52,49 +37,8 @@ interface DispatchJobInput {options?: JobDispatchOptions;}-const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;-const QSTASH_PUBLISH_MAX_RETRIES = 3;-export const jobNameSchema = z- .string()- .regex(- /^[a-z][a-z0-9]*(-[a-z0-9]+)*-job$/,- 'Job name must be kebab-case ending in "-job"',- );--function getJobsEndpointUrl(name: string) {- return `${JOBS_ENDPOINT_URL}/${name}`;-}--// QStash label: user-provided tag first, job name appended for log filtering-function buildJobLabel(name: string, label?: string) {- return label ? `${label},${name}` : name;-}--// QStash deduplicationId: user-provided id first, job name appended for cross-job isolation-function buildJobDeduplicationId(name: string, deduplicationId?: string) {- if (!deduplicationId) {- return undefined;- }-- return `${deduplicationId},${name}`;-}--// Wire format published to QStash and consumed by /api/jobs/process/[jobName]-export const jobEnvelopeSchema = z.object({- name: jobNameSchema,- dispatchedAt: z.string(),- payload: z.unknown(),-});--function toErrorFields(error: unknown) {- return {- errorName: error instanceof Error ? error.name : undefined,- errorMessage: error instanceof Error ? error.message : String(error),- };-}-async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {for (let attempt = 0; attempt <= QSTASH_PUBLISH_MAX_RETRIES; attempt++) {try {@@ -114,90 +58,6 @@ async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {throw new Error("Failed to publish to QStash.");}-type JobReplayOptions = Pick<- JobDispatchOptions,- "deduplicationId" | "retries" | "queue" | "flowControl" | "label"->;--function buildQStashJobRequest(- { name, payload, options }: DispatchJobInput,- opts?: {- dispatchedAt?: string;- batch?: boolean;- notBefore?: number;- },-) {- const envelope: JobEnvelope = {- name,- payload,- dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),- };-- const notBefore = opts?.notBefore ?? options?.notBefore;- const deduplicationId = buildJobDeduplicationId(- name,- options?.deduplicationId,- );-- return {- url: getJobsEndpointUrl(name),- body: envelope,- label: buildJobLabel(name, options?.label),- ...(options?.delay &&- opts?.notBefore === undefined && {- delay: options.delay,- }),- ...(notBefore && { notBefore }),- ...(deduplicationId && { deduplicationId }),- ...(options?.retries !== undefined && { retries: options.retries }),- ...(options?.flowControl && { flowControl: options.flowControl }),- ...(opts?.batch && options?.queue && { queueName: options.queue }),- };-}--// Used by /api/cron/queue/retry to republish deferred defineJob rows-export function buildReplayRequest(- job: {- name: string;- payload: Prisma.JsonValue;- options: Prisma.JsonValue | null;- createdAt: Date;- scheduledAt: Date | null;- },- now: Date = new Date(),-) {- const options = (job.options ?? {}) as JobReplayOptions;-- const notBefore =- job.scheduledAt && job.scheduledAt > now- ? Math.floor(job.scheduledAt.getTime() / 1000)- : undefined;-- return buildQStashJobRequest(- {- name: job.name,- payload: job.payload,- options,- },- {- dispatchedAt: job.createdAt.toISOString(),- batch: true,- notBefore,- },- );-}--export function isPublishSuccess(- response: unknown,-): response is { messageId: string } {- return (- typeof response === "object" &&- response !== null &&- "messageId" in response &&- typeof response.messageId === "string"- );-}-// Persist jobs that could not be published to QStash. The// /api/cron/queue/retry cron republishes them and deletes the rows on success.async function persistBackgroundJobs(inputs: DispatchJobInput[]) {@@ -342,7 +202,7 @@ async function dispatchJobs(logger.error("jobs.dispatch_lost", {jobName,jobCount: failedInputs.length,- ...toErrorFields(error),+ error: toErrorFields(error),});failed += failedInputs.length;@@ -364,7 +224,7 @@ async function dispatchJobs(jobName,jobCount: inputChunk.length,batch: !isSingleDispatch,- ...toErrorFields(error),+ error: toErrorFields(error),});try {@@ -375,7 +235,7 @@ async function dispatchJobs(logger.error("jobs.dispatch_lost", {jobName,jobCount: inputChunk.length,- ...toErrorFields(persistError),+ error: toErrorFields(persistError),});failed += inputChunk.length;@@ -448,113 +308,3 @@ export function defineJob<TSchema extends z.ZodType>({}export type JobDefinition = ReturnType<typeof defineJob<z.ZodType>>;--/** Republish deferred defineJob rows that failed to publish at dispatch time. */-export async function publishPendingDefineJobs(): Promise<{- attempted: number;- published: number;- failed: number;-}> {- const jobs = await prisma.job.findMany({- where: {- name: {- endsWith: "-job",- },- scheduledAt: {- lte: new Date(),- },- status: {- in: [JobStatus.pending, JobStatus.failed],- },- attempts: {- lt: MAX_JOB_ATTEMPTS,- },- },- orderBy: {- createdAt: "asc",- },- take: MAX_JOBS_PER_BATCH,- });-- if (jobs.length === 0) {- return { attempted: 0, published: 0, failed: 0 };- }-- const now = new Date();- const publishedIds: string[] = [];- const failedByError = new Map<string, string[]>();-- for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {- try {- const responses = await qstash.batchJSON(- jobChunk.map((job) => buildReplayRequest(job, now)),- );-- jobChunk.forEach((job, index) => {- if (isPublishSuccess(responses[index])) {- publishedIds.push(job.id);- } else {- const key = "QStash batch publish did not return a messageId";- const ids = failedByError.get(key) ?? [];- ids.push(job.id);- failedByError.set(key, ids);- }- });- } catch (error) {- const lastError = (- error instanceof Error ? error.message : String(error)- ).slice(0, LAST_ERROR_MAX_LENGTH);-- for (const job of jobChunk) {- const ids = failedByError.get(lastError) ?? [];- ids.push(job.id);- failedByError.set(lastError, ids);- }- }- }-- if (publishedIds.length > 0) {- await prisma.job.deleteMany({- where: {- id: {- in: publishedIds,- },- },- });- }-- for (const [lastError, ids] of failedByError) {- await prisma.job.updateMany({- where: {- id: {- in: ids,- },- },- data: {- status: JobStatus.failed,- lastError,- attempts: {- increment: 1,- },- },- });- }-- const failedIds = new Set([...failedByError.values()].flat());- const exhaustedJobs = jobs.filter(- (job) => failedIds.has(job.id) && job.attempts + 1 >= MAX_JOB_ATTEMPTS,- );-- if (exhaustedJobs.length > 0) {- logger.error("jobs.retry_exhausted", {- jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),- });- await logger.flush();- }-- return {- attempted: jobs.length,- published: publishedIds.length,- failed: failedIds.size,- };-}diff --git a/apps/web/lib/jobs/outbox.ts b/apps/web/lib/jobs/outbox.tsnew file mode 100644index 0000000000..ee8859873f--- /dev/null+++ b/apps/web/lib/jobs/outbox.ts@@ -0,0 +1,206 @@+import { logger, toErrorFields } from "@/lib/axiom/server";+import { Job, Prisma } from "@prisma/client";+import { prisma } from "../prisma";+import { MAX_JOB_ATTEMPTS, MAX_JOBS_PER_BATCH } from "./constants";+import { isDefineJobName, sendJobs } from "./send-jobs";+import { isWorkflowName, triggerWorkflows } from "./send-workflows";+import type { PersistableJob, PublishResult } from "./types";++type JobTransport = {+ matches: (name: string) => boolean;+ send: (jobs: Job[]) => Promise<PublishResult[]>;+};++const transports: JobTransport[] = [+ {+ matches: isWorkflowName,+ send: (jobs) => triggerWorkflows(jobs),+ },+ {+ matches: isDefineJobName,+ send: sendJobs,+ },+];++/** Persist jobs that failed to publish. Swallow DB errors (log only). */+export async function persistFailedJobs({+ jobs,+ failedResults,+ logEvent = "jobs.dispatch_lost",+}: {+ jobs: PersistableJob[];+ failedResults: PublishResult[];+ logEvent?: string;+}) {+ if (failedResults.length === 0) {+ return;+ }++ const failedById = new Map(+ failedResults.map((result) => [result.id, result]),+ );++ const data = jobs+ .filter((job) => failedById.has(job.id))+ .map((job) => {+ const failed = failedById.get(job.id)!;++ return {+ id: job.id,+ name: job.name,+ payload: job.payload as Prisma.InputJsonValue,+ options: (job.options ?? {}) as Prisma.InputJsonValue,+ scheduledAt: job.scheduledAt ?? new Date(),+ lastError: failed.lastError,+ attempts: 1,+ };+ });++ try {+ await prisma.job.createMany({+ skipDuplicates: true,+ data,+ });+ } catch (error) {+ logger.error(logEvent, {+ jobCount: data.length,+ error: toErrorFields(error),+ });+ await logger.flush();+ }+}++/** Delete successfully republished rows; bump attempts on failures. */+export async function settlePublishResults({+ results,+ jobs,+}: {+ results: PublishResult[];+ jobs: Pick<Job, "id" | "name" | "attempts">[];+}) {+ const publishedIds = results+ .filter((result) => result.status === "published")+ .map((result) => result.id);++ const failedResults = results.filter((result) => result.status === "failed");++ if (publishedIds.length > 0) {+ await prisma.job.deleteMany({+ where: {+ id: {+ in: publishedIds,+ },+ },+ });+ }++ if (failedResults.length === 0) {+ return {+ published: publishedIds.length,+ failed: 0,+ };+ }++ const byError = new Map<string, string[]>();++ for (const result of failedResults) {+ const key = result.lastError ?? "";+ const ids = byError.get(key) ?? [];+ ids.push(result.id);+ byError.set(key, ids);+ }++ await Promise.all(+ Array.from(byError.entries()).map(([lastError, ids]) =>+ prisma.job.updateMany({+ where: {+ id: {+ in: ids,+ },+ },+ data: {+ lastError,+ attempts: {+ increment: 1,+ },+ },+ }),+ ),+ );++ const exhaustedJobs = jobs.filter(+ (job) =>+ failedResults.some((result) => result.id === job.id) &&+ job.attempts + 1 >= MAX_JOB_ATTEMPTS,+ );++ if (exhaustedJobs.length > 0) {+ logger.error("jobs.retry_exhausted", {+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),+ });+ await logger.flush();+ }++ return {+ published: publishedIds.length,+ failed: failedResults.length,+ };+}++/** Select due rows once (indexed), send via matching transport, then settle. */+export async function publishPendingJobs(): Promise<{+ attempted: number;+ published: number;+ failed: number;+}> {+ const jobs = await prisma.job.findMany({+ where: {+ scheduledAt: {+ lte: new Date(),+ },+ attempts: {+ lt: MAX_JOB_ATTEMPTS,+ },+ },+ orderBy: {+ createdAt: "asc",+ },+ take: MAX_JOBS_PER_BATCH,+ });++ if (jobs.length === 0) {+ return { attempted: 0, published: 0, failed: 0 };+ }++ const matched = new Set<string>();++ const results = (+ await Promise.all(+ transports.map((transport) => {+ const batch = jobs.filter((job) => transport.matches(job.name));+ for (const job of batch) {+ matched.add(job.id);+ }+ return transport.send(batch);+ }),+ )+ ).flat();++ for (const job of jobs) {+ if (!matched.has(job.id)) {+ logger.error("jobs.unknown_kind", { id: job.id, name: job.name });+ }+ }++ if (matched.size < jobs.length) {+ await logger.flush();+ }++ const settled = await settlePublishResults({ results, jobs });++ return {+ attempted: jobs.length,+ published: settled.published,+ failed: settled.failed,+ };+}diff --git a/apps/web/lib/jobs/publish-workflows.ts b/apps/web/lib/jobs/publish-workflows.tsindex ff45cc1922..614ce17677 100644--- a/apps/web/lib/jobs/publish-workflows.ts+++ b/apps/web/lib/jobs/publish-workflows.ts@@ -1,42 +1,11 @@import { createId } from "@/lib/api/create-id";-import { logger } from "@/lib/axiom/server";-import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";-import { Job, JobStatus, Prisma } from "@prisma/client";-import { PublishRequest } from "@upstash/qstash";-import { Client, TriggerOptions } from "@upstash/workflow";-import { prisma } from "../prisma";+import { persistFailedJobs } from "./outbox";import {- LAST_ERROR_MAX_LENGTH,- MAX_JOB_ATTEMPTS,- MAX_JOBS_PER_BATCH,- QSTASH_BATCH_CHUNK_SIZE,-} from "./constants";--const workflowClient = new Client({- baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",- token: process.env.QSTASH_TOKEN || "",- ...(process.env.VERCEL_ENV === "preview" && {- headers: {- "x-vercel-protection-bypass":- process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",- },- }),-});--const workflowPathMap = {- partnerApproved: "/api/workflows/partner-approved",- mergePartnerAccounts: "/api/workflows/merge-partner-accounts",- createPartnerCommission: "/api/workflows/create-partner-commission",-} as const;--export type WorkflowName = keyof typeof workflowPathMap;--const WORKFLOW_NAMES = Object.keys(workflowPathMap) as WorkflowName[];--type WorkflowOptions = Pick<- PublishRequest,- "label" | "deduplicationId" | "retries" | "flowControl"->;+ triggerWorkflows,+ type WorkflowName,+ type WorkflowOptions,+} from "./send-workflows";+import type { PersistableJob } from "./types";type DispatchWorkflowInput = {name: WorkflowName;@@ -44,239 +13,12 @@ type DispatchWorkflowInput = {options?: WorkflowOptions;};-type WorkflowRecord = {- id: string;- name: WorkflowName;- payload: unknown;- options: WorkflowOptions;- createdAt?: Date;-};--type PublishResult = {- id: string;- status: typeof JobStatus.published | typeof JobStatus.failed;- lastError: string | null;- workflowRunId?: string;-};--export type DispatchWorkflowsResult = {+type DispatchWorkflowsResult = {published: number;failed: number;- results: PublishResult[];+ results: Awaited<ReturnType<typeof triggerWorkflows>>;};-export type PublishPendingWorkflowsResult = {- attempted: number;- published: number;- failed: number;-};--function toErrorFields(error: unknown) {- return {- errorName: error instanceof Error ? error.name : undefined,- errorMessage: error instanceof Error ? error.message : String(error),- };-}--function isTriggerSuccess(- response: unknown,-): response is { workflowRunId: string } {- return (- typeof response === "object" &&- response !== null &&- "workflowRunId" in response &&- typeof response.workflowRunId === "string"- );-}--function buildTriggerRequest(job: WorkflowRecord): TriggerOptions {- const workflowPath = workflowPathMap[job.name];- const workflowKey = workflowPath.split("/").pop()!;-- return {- url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,- body: job.payload,- workflowRunId: job.options.deduplicationId ?? job.id,- retries: job.options.retries ?? 5,- flowControl: job.options.flowControl ?? {- key: workflowKey,- parallelism: 15,- },- ...(job.options.label && { label: job.options.label }),- };-}--async function triggerWorkflows(- jobs: WorkflowRecord[],-): Promise<PublishResult[]> {- if (jobs.length === 0) {- return [];- }-- const results: PublishResult[] = [];-- for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {- try {- const responses = await workflowClient.trigger(- jobChunk.map((job) => buildTriggerRequest(job)),- );-- jobChunk.forEach((job, index) => {- const response = responses[index];-- if (isTriggerSuccess(response)) {- results.push({- id: job.id,- status: JobStatus.published,- lastError: null,- workflowRunId: response.workflowRunId,- });- } else {- results.push({- id: job.id,- status: JobStatus.failed,- lastError: "Workflow trigger did not return a workflowRunId",- });- }- });- } catch (error) {- const lastError = serializeError(error).slice(0, LAST_ERROR_MAX_LENGTH);-- logger.error("workflows.publish_failed", {- jobCount: jobChunk.length,- ...toErrorFields(error),- });-- for (const job of jobChunk) {- results.push({- id: job.id,- status: JobStatus.failed,- lastError,- });- }- }- }-- await logger.flush();-- return results;-}--async function persistFailedWorkflows({- jobs,- failedResults,-}: {- jobs: WorkflowRecord[];- failedResults: PublishResult[];-}) {- if (failedResults.length === 0) {- return;- }-- const failedById = new Map(- failedResults.map((result) => [result.id, result]),- );-- const data = jobs- .filter((job) => failedById.has(job.id))- .map((job) => {- const failed = failedById.get(job.id)!;-- return {- id: job.id,- name: job.name,- payload: job.payload as Prisma.InputJsonValue,- options: job.options as Prisma.InputJsonValue,- scheduledAt: new Date(),- status: JobStatus.failed,- lastError: failed.lastError,- attempts: 1,- };- });-- try {- await prisma.job.createMany({- skipDuplicates: true,- data,- });- } catch (error) {- logger.error("workflows.dispatch_lost", {- jobCount: data.length,- ...toErrorFields(error),- });- await logger.flush();- }-}--async function deletePublishedWorkflows(publishedResults: PublishResult[]) {- if (publishedResults.length === 0) {- return;- }-- await prisma.job.deleteMany({- where: {- id: {- in: publishedResults.map((result) => result.id),- },- },- });-}--async function markWorkflowsAsFailed({- failedResults,- jobs,-}: {- failedResults: PublishResult[];- jobs: Pick<Job, "id" | "name" | "attempts">[];-}) {- if (failedResults.length === 0) {- return;- }-- const byError = new Map<string, string[]>();-- for (const result of failedResults) {- const key = result.lastError ?? "";- const ids = byError.get(key) ?? [];- ids.push(result.id);- byError.set(key, ids);- }-- await Promise.all(- [...byError.entries()].map(([lastError, ids]) =>- prisma.job.updateMany({- where: {- id: {- in: ids,- },- },- data: {- status: JobStatus.failed,- lastError,- attempts: {- increment: 1,- },- },- }),- ),- );-- const exhaustedJobs = jobs.filter(- (job) =>- failedResults.some((result) => result.id === job.id) &&- job.attempts + 1 >= MAX_JOB_ATTEMPTS,- );-- if (exhaustedJobs.length > 0) {- for (const { id, name } of exhaustedJobs) {- logger.error("workflows.retry_exhausted", { id, name });- }-- await logger.flush();- }-}--/** Publish workflows to QStash; persist only on failure for the retry cron. */export async function dispatchWorkflows(input: DispatchWorkflowInput | DispatchWorkflowInput[],): Promise<DispatchWorkflowsResult> {@@ -290,7 +32,7 @@ export async function dispatchWorkflows(};}- const jobs: WorkflowRecord[] = inputs.map((job) => {+ const jobs: PersistableJob[] = inputs.map((job) => {const id = createId({ prefix: "job_" });return {@@ -309,77 +51,20 @@ export async function dispatchWorkflows(const results = await triggerWorkflows(jobs);const publishedResults = results.filter(- (result) => result.status === JobStatus.published,- );- const failedResults = results.filter(- (result) => result.status === JobStatus.failed,+ (result) => result.status === "published",);- await persistFailedWorkflows({ jobs, failedResults });+ const failedResults = results.filter((result) => result.status === "failed");- return {- published: publishedResults.length,- failed: failedResults.length,- results,- };-}--/** Republish deferred workflows that failed to publish at dispatch time. */-export async function publishPendingWorkflows(): Promise<PublishPendingWorkflowsResult> {- const emptyResult: PublishPendingWorkflowsResult = {- attempted: 0,- published: 0,- failed: 0,- };-- const jobs = await prisma.job.findMany({- where: {- name: {- in: WORKFLOW_NAMES,- },- scheduledAt: {- lte: new Date(),- },- status: {- in: [JobStatus.pending, JobStatus.failed],- },- attempts: {- lt: MAX_JOB_ATTEMPTS,- },- },- orderBy: {- createdAt: "asc",- },- take: MAX_JOBS_PER_BATCH,+ await persistFailedJobs({+ jobs,+ failedResults,+ logEvent: "workflows.dispatch_lost",});- if (jobs.length === 0) {- return emptyResult;- }-- const workflowJobs: WorkflowRecord[] = jobs.map((job) => ({- id: job.id,- name: job.name as WorkflowName,- payload: job.payload,- options: (job.options ?? {}) as WorkflowOptions,- createdAt: job.createdAt,- }));-- const results = await triggerWorkflows(workflowJobs);-- const publishedResults = results.filter(- (result) => result.status === JobStatus.published,- );- const failedResults = results.filter(- (result) => result.status === JobStatus.failed,- );-- await deletePublishedWorkflows(publishedResults);- await markWorkflowsAsFailed({ failedResults, jobs });-return {- attempted: jobs.length,published: publishedResults.length,failed: failedResults.length,+ results,};}diff --git a/apps/web/lib/jobs/send-jobs.ts b/apps/web/lib/jobs/send-jobs.tsnew file mode 100644index 0000000000..45cfffb899--- /dev/null+++ b/apps/web/lib/jobs/send-jobs.ts@@ -0,0 +1,207 @@+import { logger } from "@/lib/axiom/server";+import { qstash } from "@/lib/cron";+import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";+import { Job, Prisma } from "@prisma/client";+import { PublishRequest } from "@upstash/qstash";+import * as z from "zod/v4";+import {+ LAST_ERROR_MAX_LENGTH,+ QSTASH_BATCH_CHUNK_SIZE,+} from "./constants";+import type { PublishResult } from "./types";++export type JobDispatchOptions = Pick<+ PublishRequest,+ | "delay"+ | "notBefore"+ | "deduplicationId"+ | "retries"+ | "flowControl"+ | "label"+> & {+ queue?: string;+};++type JobReplayOptions = Pick<+ JobDispatchOptions,+ "deduplicationId" | "retries" | "queue" | "flowControl" | "label"+>;++type DispatchJobInput = {+ name: string;+ payload: unknown;+ options?: JobDispatchOptions;+};++const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;++export const jobNameSchema = z+ .string()+ .regex(+ /^[a-z][a-z0-9]*(-[a-z0-9]+)*-job$/,+ 'Job name must be kebab-case ending in "-job"',+ );++export const jobEnvelopeSchema = z.object({+ name: jobNameSchema,+ dispatchedAt: z.string(),+ payload: z.unknown(),+});++type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;++export function isDefineJobName(name: string) {+ return jobNameSchema.safeParse(name).success;+}++export function getJobsEndpointUrl(name: string) {+ return `${JOBS_ENDPOINT_URL}/${name}`;+}++export function buildJobLabel(name: string, label?: string) {+ return label ? `${label},${name}` : name;+}++export function buildJobDeduplicationId(name: string, deduplicationId?: string) {+ if (!deduplicationId) {+ return undefined;+ }++ return `${deduplicationId},${name}`;+}++export function buildQStashJobRequest(+ { name, payload, options }: DispatchJobInput,+ opts?: {+ dispatchedAt?: string;+ batch?: boolean;+ notBefore?: number;+ },+) {+ const envelope: JobEnvelope = {+ name,+ payload,+ dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),+ };++ const notBefore = opts?.notBefore ?? options?.notBefore;+ const deduplicationId = buildJobDeduplicationId(+ name,+ options?.deduplicationId,+ );++ return {+ url: getJobsEndpointUrl(name),+ body: envelope,+ label: buildJobLabel(name, options?.label),+ ...(options?.delay &&+ opts?.notBefore === undefined && {+ delay: options.delay,+ }),+ ...(notBefore && { notBefore }),+ ...(deduplicationId && { deduplicationId }),+ ...(options?.retries !== undefined && { retries: options.retries }),+ ...(options?.flowControl && { flowControl: options.flowControl }),+ ...(opts?.batch && options?.queue && { queueName: options.queue }),+ };+}++export function buildReplayRequest(+ job: {+ name: string;+ payload: Prisma.JsonValue;+ options: Prisma.JsonValue | null;+ createdAt: Date;+ scheduledAt: Date | null;+ },+ now: Date = new Date(),+) {+ const options = (job.options ?? {}) as JobReplayOptions;++ const notBefore =+ job.scheduledAt && job.scheduledAt > now+ ? Math.floor(job.scheduledAt.getTime() / 1000)+ : undefined;++ return buildQStashJobRequest(+ {+ name: job.name,+ payload: job.payload,+ options,+ },+ {+ dispatchedAt: job.createdAt.toISOString(),+ batch: true,+ notBefore,+ },+ );+}++export function isPublishSuccess(+ response: unknown,+): response is { messageId: string } {+ return (+ typeof response === "object" &&+ response !== null &&+ "messageId" in response &&+ typeof response.messageId === "string"+ );+}++/** Publish defineJob rows to QStash (no DB writes). */+export async function sendJobs(jobs: Job[]): Promise<PublishResult[]> {+ if (jobs.length === 0) {+ return [];+ }++ const results: PublishResult[] = [];+ const now = new Date();++ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {+ try {+ const responses = await qstash.batchJSON(+ jobChunk.map((job) => buildReplayRequest(job, now)),+ );++ jobChunk.forEach((job, index) => {+ const response = responses[index];++ if (isPublishSuccess(response)) {+ results.push({+ id: job.id,+ status: "published",+ lastError: null,+ messageId: response.messageId,+ });+ } else {+ results.push({+ id: job.id,+ status: "failed",+ lastError: "QStash batch publish did not return a messageId",+ });+ }+ });+ } catch (error) {+ const lastError = (+ error instanceof Error ? error.message : String(error)+ ).slice(0, LAST_ERROR_MAX_LENGTH);++ logger.error("jobs.publish_failed", {+ jobCount: jobChunk.length,+ errorMessage: lastError,+ });++ for (const job of jobChunk) {+ results.push({+ id: job.id,+ status: "failed",+ lastError,+ });+ }+ }+ }++ await logger.flush();++ return results;+}diff --git a/apps/web/lib/jobs/send-workflows.ts b/apps/web/lib/jobs/send-workflows.tsnew file mode 100644index 0000000000..cde6de27e2--- /dev/null+++ b/apps/web/lib/jobs/send-workflows.ts@@ -0,0 +1,121 @@+import { logger, toErrorFields } from "@/lib/axiom/server";+import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";+import { PublishRequest } from "@upstash/qstash";+import { Client, TriggerOptions } from "@upstash/workflow";+import { LAST_ERROR_MAX_LENGTH, QSTASH_BATCH_CHUNK_SIZE } from "./constants";+import type { PersistableJob, PublishResult } from "./types";++const workflowClient = new Client({+ baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",+ token: process.env.QSTASH_TOKEN || "",+ ...(process.env.VERCEL_ENV === "preview" && {+ headers: {+ "x-vercel-protection-bypass":+ process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",+ },+ }),+});++const workflowPathMap = {+ partnerApproved: "/api/workflows/partner-approved",+ mergePartnerAccounts: "/api/workflows/merge-partner-accounts",+ createPartnerCommission: "/api/workflows/create-partner-commission",+} as const;++export type WorkflowName = keyof typeof workflowPathMap;++export type WorkflowOptions = Pick<+ PublishRequest,+ "label" | "deduplicationId" | "retries" | "flowControl"+>;++export function isWorkflowName(name: string): name is WorkflowName {+ return Object.prototype.hasOwnProperty.call(workflowPathMap, name);+}++function isTriggerSuccess(+ response: unknown,+): response is { workflowRunId: string } {+ return (+ typeof response === "object" &&+ response !== null &&+ "workflowRunId" in response &&+ typeof response.workflowRunId === "string"+ );+}++function buildTriggerRequest(job: PersistableJob): TriggerOptions {+ const name = job.name as WorkflowName;+ const options = (job.options ?? {}) as WorkflowOptions;+ const workflowPath = workflowPathMap[name];+ const workflowKey = workflowPath.split("/").pop()!;++ return {+ url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,+ body: job.payload,+ workflowRunId: options.deduplicationId ?? job.id,+ retries: options.retries ?? 5,+ flowControl: options.flowControl ?? {+ key: workflowKey,+ parallelism: 15,+ },+ ...(options.label && { label: options.label }),+ };+}++/** Publish workflows to QStash Workflows (no DB writes). */+export async function triggerWorkflows(+ jobs: PersistableJob[],+): Promise<PublishResult[]> {+ if (jobs.length === 0) {+ return [];+ }++ const results: PublishResult[] = [];++ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {+ try {+ const responses = await workflowClient.trigger(+ jobChunk.map((job) => buildTriggerRequest(job)),+ );++ jobChunk.forEach((job, index) => {+ const response = responses[index];++ if (isTriggerSuccess(response)) {+ results.push({+ id: job.id,+ status: "published",+ lastError: null,+ workflowRunId: response.workflowRunId,+ });+ } else {+ results.push({+ id: job.id,+ status: "failed",+ lastError: "Workflow trigger did not return a workflowRunId",+ });+ }+ });+ } catch (error) {+ const lastError = serializeError(error).slice(0, LAST_ERROR_MAX_LENGTH);++ logger.error("workflows.publish_failed", {+ jobCount: jobChunk.length,+ error: toErrorFields(error),+ });++ for (const job of jobChunk) {+ results.push({+ id: job.id,+ status: "failed",+ lastError,+ });+ }+ }+ }++ await logger.flush();++ return results;+}diff --git a/apps/web/lib/jobs/types.ts b/apps/web/lib/jobs/types.tsnew file mode 100644index 0000000000..769d18d8ff--- /dev/null+++ b/apps/web/lib/jobs/types.ts@@ -0,0 +1,15 @@+export type PublishResult = {+ id: string;+ status: "published" | "failed";+ lastError: string | null;+ messageId?: string;+ workflowRunId?: string;+};++export type PersistableJob = {+ id: string;+ name: string;+ payload: unknown;+ options?: unknown;+ scheduledAt?: Date;+};diff --git a/apps/web/prisma/schema/job.prisma b/apps/web/prisma/schema/job.prismaindex 779b3b5859..3d62641121 100644--- a/apps/web/prisma/schema/job.prisma+++ b/apps/web/prisma/schema/job.prisma@@ -1,21 +1,15 @@-enum JobStatus {- pending- published- failed-}-+// Background jobs that could not be published to QStash (e.g. QStash outage).+// The /api/cron/queue/retry cron republishes these and deletes them on success.model Job {- id String @id+ id String @idname Stringpayload Jsonoptions Json? // dispatch options replayed verbatim: deduplicationId, retries, queue, flowControl, labelscheduledAt DateTime @default(now()) // when we should publish the job- status JobStatus @default(pending)- attempts Int @default(0)- lastError String? @db.Text- createdAt DateTime @default(now())- updatedAt DateTime @updatedAt+ attempts Int @default(0)+ lastError String? @db.Text+ createdAt DateTime @default(now())+ updatedAt DateTime @updatedAt- @@index([status, scheduledAt, attempts]) // retry cron: due pending/failed jobs- @@index([status, createdAt]) // daily cleanup by status + age+ @@index([scheduledAt, attempts]) // retry cron: due jobs under the attempt cap}