Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Split job publishing into shared outbox transports and drop Job status.

Kiran K383f7b4433c6
diff --git a/apps/web/app/(ee)/api/cron/queue/retry/route.ts b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
index f16bab8611..f0f09985b4 100644
--- a/apps/web/app/(ee)/api/cron/queue/retry/route.ts
+++ b/apps/web/app/(ee)/api/cron/queue/retry/route.ts
@@ -1,6 +1,5 @@
import { withCron } from "@/lib/cron/with-cron";
-import { publishPendingDefineJobs } from "@/lib/jobs";
-import { publishPendingWorkflows } from "@/lib/jobs/publish-workflows";
+import { publishPendingJobs } from "@/lib/jobs/outbox";
import { redis } from "@/lib/upstash/redis";
import { logAndRespond } from "../../utils";
@@ -12,8 +11,8 @@ export const dynamic = "force-dynamic";
const LOCK_KEY = "lock:queue-retry";
const LOCK_TTL_SECONDS = 600;
-// GET /api/cron/queue/retry – republish workflows / defineJob rows that failed
-// to publish to QStash at dispatch time; rows are deleted on successful publish
+// GET /api/cron/queue/retry – republish jobs that failed to publish to QStash
+// at dispatch time; rows are deleted on successful publish
export const GET = withCron(async () => {
const acquired = await redis.set(LOCK_KEY, "1", {
nx: true,
@@ -27,14 +26,7 @@ export const GET = withCron(async () => {
}
try {
- const [workflows, defineJobs] = await Promise.all([
- publishPendingWorkflows(),
- publishPendingDefineJobs(),
- ]);
-
- const attempted = workflows.attempted + defineJobs.attempted;
- const published = workflows.published + defineJobs.published;
- const failed = workflows.failed + defineJobs.failed;
+ const { attempted, published, failed } = await publishPendingJobs();
if (attempted === 0) {
return logAndRespond("No background jobs to retry.");
diff --git a/apps/web/app/api/jobs/process/[jobName]/route.ts b/apps/web/app/api/jobs/process/[jobName]/route.ts
index 57c5001983..3b0e2f46c2 100644
--- a/apps/web/app/api/jobs/process/[jobName]/route.ts
+++ b/apps/web/app/api/jobs/process/[jobName]/route.ts
@@ -1,7 +1,7 @@
import { handleAndReturnErrorResponse } from "@/lib/api/errors";
import { logger, withAxiomBodyLog } from "@/lib/axiom/server";
import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
-import { jobEnvelopeSchema } from "@/lib/jobs";
+import { jobEnvelopeSchema } from "@/lib/jobs/send-jobs";
import { loadJob } from "@/lib/jobs/registry";
import * as z from "zod/v4";
diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.ts
index 1d09986b5e..94650c516b 100644
--- a/apps/web/lib/jobs/index.ts
+++ b/apps/web/lib/jobs/index.ts
@@ -1,33 +1,18 @@
import { createId } from "@/lib/api/create-id";
-import { logger } from "@/lib/axiom/server";
+import { logger, toErrorFields } from "@/lib/axiom/server";
import { qstash } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
-import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";
-import { JobStatus, Prisma } from "@prisma/client";
-import { PublishRequest } from "@upstash/qstash";
+import { chunk } from "@dub/utils";
+import { Prisma } from "@prisma/client";
import * as z from "zod/v4";
+import { QSTASH_BATCH_CHUNK_SIZE } from "./constants";
import {
- LAST_ERROR_MAX_LENGTH,
- MAX_JOB_ATTEMPTS,
- MAX_JOBS_PER_BATCH,
- QSTASH_BATCH_CHUNK_SIZE,
-} from "./constants";
-
-type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;
-
-type JobPublishOptions = Pick<
- PublishRequest,
- | "delay"
- | "notBefore"
- | "deduplicationId"
- | "retries"
- | "flowControl"
- | "label"
->;
-
-export type JobDispatchOptions = JobPublishOptions & {
- queue?: string;
-};
+ buildJobLabel,
+ buildQStashJobRequest,
+ isPublishSuccess,
+ jobNameSchema,
+ type JobDispatchOptions,
+} from "./send-jobs";
// Per-job defaults, merged under per-dispatch options
export type JobDefaults = Pick<
@@ -52,49 +37,8 @@ interface DispatchJobInput {
options?: JobDispatchOptions;
}
-const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;
-
const QSTASH_PUBLISH_MAX_RETRIES = 3;
-export const jobNameSchema = z
- .string()
- .regex(
- /^[a-z][a-z0-9]*(-[a-z0-9]+)*-job$/,
- 'Job name must be kebab-case ending in "-job"',
- );
-
-function getJobsEndpointUrl(name: string) {
- return `${JOBS_ENDPOINT_URL}/${name}`;
-}
-
-// QStash label: user-provided tag first, job name appended for log filtering
-function buildJobLabel(name: string, label?: string) {
- return label ? `${label},${name}` : name;
-}
-
-// QStash deduplicationId: user-provided id first, job name appended for cross-job isolation
-function buildJobDeduplicationId(name: string, deduplicationId?: string) {
- if (!deduplicationId) {
- return undefined;
- }
-
- return `${deduplicationId},${name}`;
-}
-
-// Wire format published to QStash and consumed by /api/jobs/process/[jobName]
-export const jobEnvelopeSchema = z.object({
- name: jobNameSchema,
- dispatchedAt: z.string(),
- payload: z.unknown(),
-});
-
-function toErrorFields(error: unknown) {
- return {
- errorName: error instanceof Error ? error.name : undefined,
- errorMessage: error instanceof Error ? error.message : String(error),
- };
-}
-
async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {
for (let attempt = 0; attempt <= QSTASH_PUBLISH_MAX_RETRIES; attempt++) {
try {
@@ -114,90 +58,6 @@ async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {
throw new Error("Failed to publish to QStash.");
}
-type JobReplayOptions = Pick<
- JobDispatchOptions,
- "deduplicationId" | "retries" | "queue" | "flowControl" | "label"
->;
-
-function buildQStashJobRequest(
- { name, payload, options }: DispatchJobInput,
- opts?: {
- dispatchedAt?: string;
- batch?: boolean;
- notBefore?: number;
- },
-) {
- const envelope: JobEnvelope = {
- name,
- payload,
- dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),
- };
-
- const notBefore = opts?.notBefore ?? options?.notBefore;
- const deduplicationId = buildJobDeduplicationId(
- name,
- options?.deduplicationId,
- );
-
- return {
- url: getJobsEndpointUrl(name),
- body: envelope,
- label: buildJobLabel(name, options?.label),
- ...(options?.delay &&
- opts?.notBefore === undefined && {
- delay: options.delay,
- }),
- ...(notBefore && { notBefore }),
- ...(deduplicationId && { deduplicationId }),
- ...(options?.retries !== undefined && { retries: options.retries }),
- ...(options?.flowControl && { flowControl: options.flowControl }),
- ...(opts?.batch && options?.queue && { queueName: options.queue }),
- };
-}
-
-// Used by /api/cron/queue/retry to republish deferred defineJob rows
-export function buildReplayRequest(
- job: {
- name: string;
- payload: Prisma.JsonValue;
- options: Prisma.JsonValue | null;
- createdAt: Date;
- scheduledAt: Date | null;
- },
- now: Date = new Date(),
-) {
- const options = (job.options ?? {}) as JobReplayOptions;
-
- const notBefore =
- job.scheduledAt && job.scheduledAt > now
- ? Math.floor(job.scheduledAt.getTime() / 1000)
- : undefined;
-
- return buildQStashJobRequest(
- {
- name: job.name,
- payload: job.payload,
- options,
- },
- {
- dispatchedAt: job.createdAt.toISOString(),
- batch: true,
- notBefore,
- },
- );
-}
-
-export function isPublishSuccess(
- response: unknown,
-): response is { messageId: string } {
- return (
- typeof response === "object" &&
- response !== null &&
- "messageId" in response &&
- typeof response.messageId === "string"
- );
-}
-
// Persist jobs that could not be published to QStash. The
// /api/cron/queue/retry cron republishes them and deletes the rows on success.
async function persistBackgroundJobs(inputs: DispatchJobInput[]) {
@@ -342,7 +202,7 @@ async function dispatchJobs(
logger.error("jobs.dispatch_lost", {
jobName,
jobCount: failedInputs.length,
- ...toErrorFields(error),
+ error: toErrorFields(error),
});
failed += failedInputs.length;
@@ -364,7 +224,7 @@ async function dispatchJobs(
jobName,
jobCount: inputChunk.length,
batch: !isSingleDispatch,
- ...toErrorFields(error),
+ error: toErrorFields(error),
});
try {
@@ -375,7 +235,7 @@ async function dispatchJobs(
logger.error("jobs.dispatch_lost", {
jobName,
jobCount: inputChunk.length,
- ...toErrorFields(persistError),
+ error: toErrorFields(persistError),
});
failed += inputChunk.length;
@@ -448,113 +308,3 @@ export function defineJob<TSchema extends z.ZodType>({
}
export type JobDefinition = ReturnType<typeof defineJob<z.ZodType>>;
-
-/** Republish deferred defineJob rows that failed to publish at dispatch time. */
-export async function publishPendingDefineJobs(): Promise<{
- attempted: number;
- published: number;
- failed: number;
-}> {
- const jobs = await prisma.job.findMany({
- where: {
- name: {
- endsWith: "-job",
- },
- scheduledAt: {
- lte: new Date(),
- },
- status: {
- in: [JobStatus.pending, JobStatus.failed],
- },
- attempts: {
- lt: MAX_JOB_ATTEMPTS,
- },
- },
- orderBy: {
- createdAt: "asc",
- },
- take: MAX_JOBS_PER_BATCH,
- });
-
- if (jobs.length === 0) {
- return { attempted: 0, published: 0, failed: 0 };
- }
-
- const now = new Date();
- const publishedIds: string[] = [];
- const failedByError = new Map<string, string[]>();
-
- for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
- try {
- const responses = await qstash.batchJSON(
- jobChunk.map((job) => buildReplayRequest(job, now)),
- );
-
- jobChunk.forEach((job, index) => {
- if (isPublishSuccess(responses[index])) {
- publishedIds.push(job.id);
- } else {
- const key = "QStash batch publish did not return a messageId";
- const ids = failedByError.get(key) ?? [];
- ids.push(job.id);
- failedByError.set(key, ids);
- }
- });
- } catch (error) {
- const lastError = (
- error instanceof Error ? error.message : String(error)
- ).slice(0, LAST_ERROR_MAX_LENGTH);
-
- for (const job of jobChunk) {
- const ids = failedByError.get(lastError) ?? [];
- ids.push(job.id);
- failedByError.set(lastError, ids);
- }
- }
- }
-
- if (publishedIds.length > 0) {
- await prisma.job.deleteMany({
- where: {
- id: {
- in: publishedIds,
- },
- },
- });
- }
-
- for (const [lastError, ids] of failedByError) {
- await prisma.job.updateMany({
- where: {
- id: {
- in: ids,
- },
- },
- data: {
- status: JobStatus.failed,
- lastError,
- attempts: {
- increment: 1,
- },
- },
- });
- }
-
- const failedIds = new Set([...failedByError.values()].flat());
- const exhaustedJobs = jobs.filter(
- (job) => failedIds.has(job.id) && job.attempts + 1 >= MAX_JOB_ATTEMPTS,
- );
-
- if (exhaustedJobs.length > 0) {
- logger.error("jobs.retry_exhausted", {
- jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
- });
- await logger.flush();
- }
-
- return {
- attempted: jobs.length,
- published: publishedIds.length,
- failed: failedIds.size,
- };
-}
diff --git a/apps/web/lib/jobs/outbox.ts b/apps/web/lib/jobs/outbox.ts
new file mode 100644
index 0000000000..ee8859873f
--- /dev/null
+++ b/apps/web/lib/jobs/outbox.ts
@@ -0,0 +1,206 @@
+import { logger, toErrorFields } from "@/lib/axiom/server";
+import { Job, Prisma } from "@prisma/client";
+import { prisma } from "../prisma";
+import { MAX_JOB_ATTEMPTS, MAX_JOBS_PER_BATCH } from "./constants";
+import { isDefineJobName, sendJobs } from "./send-jobs";
+import { isWorkflowName, triggerWorkflows } from "./send-workflows";
+import type { PersistableJob, PublishResult } from "./types";
+
+type JobTransport = {
+ matches: (name: string) => boolean;
+ send: (jobs: Job[]) => Promise<PublishResult[]>;
+};
+
+const transports: JobTransport[] = [
+ {
+ matches: isWorkflowName,
+ send: (jobs) => triggerWorkflows(jobs),
+ },
+ {
+ matches: isDefineJobName,
+ send: sendJobs,
+ },
+];
+
+/** Persist jobs that failed to publish. Swallow DB errors (log only). */
+export async function persistFailedJobs({
+ jobs,
+ failedResults,
+ logEvent = "jobs.dispatch_lost",
+}: {
+ jobs: PersistableJob[];
+ failedResults: PublishResult[];
+ logEvent?: string;
+}) {
+ if (failedResults.length === 0) {
+ return;
+ }
+
+ const failedById = new Map(
+ failedResults.map((result) => [result.id, result]),
+ );
+
+ const data = jobs
+ .filter((job) => failedById.has(job.id))
+ .map((job) => {
+ const failed = failedById.get(job.id)!;
+
+ return {
+ id: job.id,
+ name: job.name,
+ payload: job.payload as Prisma.InputJsonValue,
+ options: (job.options ?? {}) as Prisma.InputJsonValue,
+ scheduledAt: job.scheduledAt ?? new Date(),
+ lastError: failed.lastError,
+ attempts: 1,
+ };
+ });
+
+ try {
+ await prisma.job.createMany({
+ skipDuplicates: true,
+ data,
+ });
+ } catch (error) {
+ logger.error(logEvent, {
+ jobCount: data.length,
+ error: toErrorFields(error),
+ });
+ await logger.flush();
+ }
+}
+
+/** Delete successfully republished rows; bump attempts on failures. */
+export async function settlePublishResults({
+ results,
+ jobs,
+}: {
+ results: PublishResult[];
+ jobs: Pick<Job, "id" | "name" | "attempts">[];
+}) {
+ const publishedIds = results
+ .filter((result) => result.status === "published")
+ .map((result) => result.id);
+
+ const failedResults = results.filter((result) => result.status === "failed");
+
+ if (publishedIds.length > 0) {
+ await prisma.job.deleteMany({
+ where: {
+ id: {
+ in: publishedIds,
+ },
+ },
+ });
+ }
+
+ if (failedResults.length === 0) {
+ return {
+ published: publishedIds.length,
+ failed: 0,
+ };
+ }
+
+ const byError = new Map<string, string[]>();
+
+ for (const result of failedResults) {
+ const key = result.lastError ?? "";
+ const ids = byError.get(key) ?? [];
+ ids.push(result.id);
+ byError.set(key, ids);
+ }
+
+ await Promise.all(
+ Array.from(byError.entries()).map(([lastError, ids]) =>
+ prisma.job.updateMany({
+ where: {
+ id: {
+ in: ids,
+ },
+ },
+ data: {
+ lastError,
+ attempts: {
+ increment: 1,
+ },
+ },
+ }),
+ ),
+ );
+
+ const exhaustedJobs = jobs.filter(
+ (job) =>
+ failedResults.some((result) => result.id === job.id) &&
+ job.attempts + 1 >= MAX_JOB_ATTEMPTS,
+ );
+
+ if (exhaustedJobs.length > 0) {
+ logger.error("jobs.retry_exhausted", {
+ jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),
+ });
+ await logger.flush();
+ }
+
+ return {
+ published: publishedIds.length,
+ failed: failedResults.length,
+ };
+}
+
+/** Select due rows once (indexed), send via matching transport, then settle. */
+export async function publishPendingJobs(): Promise<{
+ attempted: number;
+ published: number;
+ failed: number;
+}> {
+ const jobs = await prisma.job.findMany({
+ where: {
+ scheduledAt: {
+ lte: new Date(),
+ },
+ attempts: {
+ lt: MAX_JOB_ATTEMPTS,
+ },
+ },
+ orderBy: {
+ createdAt: "asc",
+ },
+ take: MAX_JOBS_PER_BATCH,
+ });
+
+ if (jobs.length === 0) {
+ return { attempted: 0, published: 0, failed: 0 };
+ }
+
+ const matched = new Set<string>();
+
+ const results = (
+ await Promise.all(
+ transports.map((transport) => {
+ const batch = jobs.filter((job) => transport.matches(job.name));
+ for (const job of batch) {
+ matched.add(job.id);
+ }
+ return transport.send(batch);
+ }),
+ )
+ ).flat();
+
+ for (const job of jobs) {
+ if (!matched.has(job.id)) {
+ logger.error("jobs.unknown_kind", { id: job.id, name: job.name });
+ }
+ }
+
+ if (matched.size < jobs.length) {
+ await logger.flush();
+ }
+
+ const settled = await settlePublishResults({ results, jobs });
+
+ return {
+ attempted: jobs.length,
+ published: settled.published,
+ failed: settled.failed,
+ };
+}
diff --git a/apps/web/lib/jobs/publish-workflows.ts b/apps/web/lib/jobs/publish-workflows.ts
index ff45cc1922..614ce17677 100644
--- a/apps/web/lib/jobs/publish-workflows.ts
+++ b/apps/web/lib/jobs/publish-workflows.ts
@@ -1,42 +1,11 @@
import { createId } from "@/lib/api/create-id";
-import { logger } from "@/lib/axiom/server";
-import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
-import { Job, JobStatus, Prisma } from "@prisma/client";
-import { PublishRequest } from "@upstash/qstash";
-import { Client, TriggerOptions } from "@upstash/workflow";
-import { prisma } from "../prisma";
+import { persistFailedJobs } from "./outbox";
import {
- LAST_ERROR_MAX_LENGTH,
- MAX_JOB_ATTEMPTS,
- MAX_JOBS_PER_BATCH,
- QSTASH_BATCH_CHUNK_SIZE,
-} from "./constants";
-
-const workflowClient = new Client({
- baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",
- token: process.env.QSTASH_TOKEN || "",
- ...(process.env.VERCEL_ENV === "preview" && {
- headers: {
- "x-vercel-protection-bypass":
- process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",
- },
- }),
-});
-
-const workflowPathMap = {
- partnerApproved: "/api/workflows/partner-approved",
- mergePartnerAccounts: "/api/workflows/merge-partner-accounts",
- createPartnerCommission: "/api/workflows/create-partner-commission",
-} as const;
-
-export type WorkflowName = keyof typeof workflowPathMap;
-
-const WORKFLOW_NAMES = Object.keys(workflowPathMap) as WorkflowName[];
-
-type WorkflowOptions = Pick<
- PublishRequest,
- "label" | "deduplicationId" | "retries" | "flowControl"
->;
+ triggerWorkflows,
+ type WorkflowName,
+ type WorkflowOptions,
+} from "./send-workflows";
+import type { PersistableJob } from "./types";
type DispatchWorkflowInput = {
name: WorkflowName;
@@ -44,239 +13,12 @@ type DispatchWorkflowInput = {
options?: WorkflowOptions;
};
-type WorkflowRecord = {
- id: string;
- name: WorkflowName;
- payload: unknown;
- options: WorkflowOptions;
- createdAt?: Date;
-};
-
-type PublishResult = {
- id: string;
- status: typeof JobStatus.published | typeof JobStatus.failed;
- lastError: string | null;
- workflowRunId?: string;
-};
-
-export type DispatchWorkflowsResult = {
+type DispatchWorkflowsResult = {
published: number;
failed: number;
- results: PublishResult[];
+ results: Awaited<ReturnType<typeof triggerWorkflows>>;
};
-export type PublishPendingWorkflowsResult = {
- attempted: number;
- published: number;
- failed: number;
-};
-
-function toErrorFields(error: unknown) {
- return {
- errorName: error instanceof Error ? error.name : undefined,
- errorMessage: error instanceof Error ? error.message : String(error),
- };
-}
-
-function isTriggerSuccess(
- response: unknown,
-): response is { workflowRunId: string } {
- return (
- typeof response === "object" &&
- response !== null &&
- "workflowRunId" in response &&
- typeof response.workflowRunId === "string"
- );
-}
-
-function buildTriggerRequest(job: WorkflowRecord): TriggerOptions {
- const workflowPath = workflowPathMap[job.name];
- const workflowKey = workflowPath.split("/").pop()!;
-
- return {
- url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,
- body: job.payload,
- workflowRunId: job.options.deduplicationId ?? job.id,
- retries: job.options.retries ?? 5,
- flowControl: job.options.flowControl ?? {
- key: workflowKey,
- parallelism: 15,
- },
- ...(job.options.label && { label: job.options.label }),
- };
-}
-
-async function triggerWorkflows(
- jobs: WorkflowRecord[],
-): Promise<PublishResult[]> {
- if (jobs.length === 0) {
- return [];
- }
-
- const results: PublishResult[] = [];
-
- for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
- try {
- const responses = await workflowClient.trigger(
- jobChunk.map((job) => buildTriggerRequest(job)),
- );
-
- jobChunk.forEach((job, index) => {
- const response = responses[index];
-
- if (isTriggerSuccess(response)) {
- results.push({
- id: job.id,
- status: JobStatus.published,
- lastError: null,
- workflowRunId: response.workflowRunId,
- });
- } else {
- results.push({
- id: job.id,
- status: JobStatus.failed,
- lastError: "Workflow trigger did not return a workflowRunId",
- });
- }
- });
- } catch (error) {
- const lastError = serializeError(error).slice(0, LAST_ERROR_MAX_LENGTH);
-
- logger.error("workflows.publish_failed", {
- jobCount: jobChunk.length,
- ...toErrorFields(error),
- });
-
- for (const job of jobChunk) {
- results.push({
- id: job.id,
- status: JobStatus.failed,
- lastError,
- });
- }
- }
- }
-
- await logger.flush();
-
- return results;
-}
-
-async function persistFailedWorkflows({
- jobs,
- failedResults,
-}: {
- jobs: WorkflowRecord[];
- failedResults: PublishResult[];
-}) {
- if (failedResults.length === 0) {
- return;
- }
-
- const failedById = new Map(
- failedResults.map((result) => [result.id, result]),
- );
-
- const data = jobs
- .filter((job) => failedById.has(job.id))
- .map((job) => {
- const failed = failedById.get(job.id)!;
-
- return {
- id: job.id,
- name: job.name,
- payload: job.payload as Prisma.InputJsonValue,
- options: job.options as Prisma.InputJsonValue,
- scheduledAt: new Date(),
- status: JobStatus.failed,
- lastError: failed.lastError,
- attempts: 1,
- };
- });
-
- try {
- await prisma.job.createMany({
- skipDuplicates: true,
- data,
- });
- } catch (error) {
- logger.error("workflows.dispatch_lost", {
- jobCount: data.length,
- ...toErrorFields(error),
- });
- await logger.flush();
- }
-}
-
-async function deletePublishedWorkflows(publishedResults: PublishResult[]) {
- if (publishedResults.length === 0) {
- return;
- }
-
- await prisma.job.deleteMany({
- where: {
- id: {
- in: publishedResults.map((result) => result.id),
- },
- },
- });
-}
-
-async function markWorkflowsAsFailed({
- failedResults,
- jobs,
-}: {
- failedResults: PublishResult[];
- jobs: Pick<Job, "id" | "name" | "attempts">[];
-}) {
- if (failedResults.length === 0) {
- return;
- }
-
- const byError = new Map<string, string[]>();
-
- for (const result of failedResults) {
- const key = result.lastError ?? "";
- const ids = byError.get(key) ?? [];
- ids.push(result.id);
- byError.set(key, ids);
- }
-
- await Promise.all(
- [...byError.entries()].map(([lastError, ids]) =>
- prisma.job.updateMany({
- where: {
- id: {
- in: ids,
- },
- },
- data: {
- status: JobStatus.failed,
- lastError,
- attempts: {
- increment: 1,
- },
- },
- }),
- ),
- );
-
- const exhaustedJobs = jobs.filter(
- (job) =>
- failedResults.some((result) => result.id === job.id) &&
- job.attempts + 1 >= MAX_JOB_ATTEMPTS,
- );
-
- if (exhaustedJobs.length > 0) {
- for (const { id, name } of exhaustedJobs) {
- logger.error("workflows.retry_exhausted", { id, name });
- }
-
- await logger.flush();
- }
-}
-
-/** Publish workflows to QStash; persist only on failure for the retry cron. */
export async function dispatchWorkflows(
input: DispatchWorkflowInput | DispatchWorkflowInput[],
): Promise<DispatchWorkflowsResult> {
@@ -290,7 +32,7 @@ export async function dispatchWorkflows(
};
}
- const jobs: WorkflowRecord[] = inputs.map((job) => {
+ const jobs: PersistableJob[] = inputs.map((job) => {
const id = createId({ prefix: "job_" });
return {
@@ -309,77 +51,20 @@ export async function dispatchWorkflows(
const results = await triggerWorkflows(jobs);
const publishedResults = results.filter(
- (result) => result.status === JobStatus.published,
- );
- const failedResults = results.filter(
- (result) => result.status === JobStatus.failed,
+ (result) => result.status === "published",
);
- await persistFailedWorkflows({ jobs, failedResults });
+ const failedResults = results.filter((result) => result.status === "failed");
- return {
- published: publishedResults.length,
- failed: failedResults.length,
- results,
- };
-}
-
-/** Republish deferred workflows that failed to publish at dispatch time. */
-export async function publishPendingWorkflows(): Promise<PublishPendingWorkflowsResult> {
- const emptyResult: PublishPendingWorkflowsResult = {
- attempted: 0,
- published: 0,
- failed: 0,
- };
-
- const jobs = await prisma.job.findMany({
- where: {
- name: {
- in: WORKFLOW_NAMES,
- },
- scheduledAt: {
- lte: new Date(),
- },
- status: {
- in: [JobStatus.pending, JobStatus.failed],
- },
- attempts: {
- lt: MAX_JOB_ATTEMPTS,
- },
- },
- orderBy: {
- createdAt: "asc",
- },
- take: MAX_JOBS_PER_BATCH,
+ await persistFailedJobs({
+ jobs,
+ failedResults,
+ logEvent: "workflows.dispatch_lost",
});
- if (jobs.length === 0) {
- return emptyResult;
- }
-
- const workflowJobs: WorkflowRecord[] = jobs.map((job) => ({
- id: job.id,
- name: job.name as WorkflowName,
- payload: job.payload,
- options: (job.options ?? {}) as WorkflowOptions,
- createdAt: job.createdAt,
- }));
-
- const results = await triggerWorkflows(workflowJobs);
-
- const publishedResults = results.filter(
- (result) => result.status === JobStatus.published,
- );
- const failedResults = results.filter(
- (result) => result.status === JobStatus.failed,
- );
-
- await deletePublishedWorkflows(publishedResults);
- await markWorkflowsAsFailed({ failedResults, jobs });
-
return {
- attempted: jobs.length,
published: publishedResults.length,
failed: failedResults.length,
+ results,
};
}
diff --git a/apps/web/lib/jobs/send-jobs.ts b/apps/web/lib/jobs/send-jobs.ts
new file mode 100644
index 0000000000..45cfffb899
--- /dev/null
+++ b/apps/web/lib/jobs/send-jobs.ts
@@ -0,0 +1,207 @@
+import { logger } from "@/lib/axiom/server";
+import { qstash } from "@/lib/cron";
+import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";
+import { Job, Prisma } from "@prisma/client";
+import { PublishRequest } from "@upstash/qstash";
+import * as z from "zod/v4";
+import {
+ LAST_ERROR_MAX_LENGTH,
+ QSTASH_BATCH_CHUNK_SIZE,
+} from "./constants";
+import type { PublishResult } from "./types";
+
+export type JobDispatchOptions = Pick<
+ PublishRequest,
+ | "delay"
+ | "notBefore"
+ | "deduplicationId"
+ | "retries"
+ | "flowControl"
+ | "label"
+> & {
+ queue?: string;
+};
+
+type JobReplayOptions = Pick<
+ JobDispatchOptions,
+ "deduplicationId" | "retries" | "queue" | "flowControl" | "label"
+>;
+
+type DispatchJobInput = {
+ name: string;
+ payload: unknown;
+ options?: JobDispatchOptions;
+};
+
+const JOBS_ENDPOINT_URL = `${APP_DOMAIN_WITH_NGROK}/api/jobs/process`;
+
+export const jobNameSchema = z
+ .string()
+ .regex(
+ /^[a-z][a-z0-9]*(-[a-z0-9]+)*-job$/,
+ 'Job name must be kebab-case ending in "-job"',
+ );
+
+export const jobEnvelopeSchema = z.object({
+ name: jobNameSchema,
+ dispatchedAt: z.string(),
+ payload: z.unknown(),
+});
+
+type JobEnvelope = z.infer<typeof jobEnvelopeSchema>;
+
+export function isDefineJobName(name: string) {
+ return jobNameSchema.safeParse(name).success;
+}
+
+export function getJobsEndpointUrl(name: string) {
+ return `${JOBS_ENDPOINT_URL}/${name}`;
+}
+
+export function buildJobLabel(name: string, label?: string) {
+ return label ? `${label},${name}` : name;
+}
+
+export function buildJobDeduplicationId(name: string, deduplicationId?: string) {
+ if (!deduplicationId) {
+ return undefined;
+ }
+
+ return `${deduplicationId},${name}`;
+}
+
+export function buildQStashJobRequest(
+ { name, payload, options }: DispatchJobInput,
+ opts?: {
+ dispatchedAt?: string;
+ batch?: boolean;
+ notBefore?: number;
+ },
+) {
+ const envelope: JobEnvelope = {
+ name,
+ payload,
+ dispatchedAt: opts?.dispatchedAt ?? new Date().toISOString(),
+ };
+
+ const notBefore = opts?.notBefore ?? options?.notBefore;
+ const deduplicationId = buildJobDeduplicationId(
+ name,
+ options?.deduplicationId,
+ );
+
+ return {
+ url: getJobsEndpointUrl(name),
+ body: envelope,
+ label: buildJobLabel(name, options?.label),
+ ...(options?.delay &&
+ opts?.notBefore === undefined && {
+ delay: options.delay,
+ }),
+ ...(notBefore && { notBefore }),
+ ...(deduplicationId && { deduplicationId }),
+ ...(options?.retries !== undefined && { retries: options.retries }),
+ ...(options?.flowControl && { flowControl: options.flowControl }),
+ ...(opts?.batch && options?.queue && { queueName: options.queue }),
+ };
+}
+
+export function buildReplayRequest(
+ job: {
+ name: string;
+ payload: Prisma.JsonValue;
+ options: Prisma.JsonValue | null;
+ createdAt: Date;
+ scheduledAt: Date | null;
+ },
+ now: Date = new Date(),
+) {
+ const options = (job.options ?? {}) as JobReplayOptions;
+
+ const notBefore =
+ job.scheduledAt && job.scheduledAt > now
+ ? Math.floor(job.scheduledAt.getTime() / 1000)
+ : undefined;
+
+ return buildQStashJobRequest(
+ {
+ name: job.name,
+ payload: job.payload,
+ options,
+ },
+ {
+ dispatchedAt: job.createdAt.toISOString(),
+ batch: true,
+ notBefore,
+ },
+ );
+}
+
+export function isPublishSuccess(
+ response: unknown,
+): response is { messageId: string } {
+ return (
+ typeof response === "object" &&
+ response !== null &&
+ "messageId" in response &&
+ typeof response.messageId === "string"
+ );
+}
+
+/** Publish defineJob rows to QStash (no DB writes). */
+export async function sendJobs(jobs: Job[]): Promise<PublishResult[]> {
+ if (jobs.length === 0) {
+ return [];
+ }
+
+ const results: PublishResult[] = [];
+ const now = new Date();
+
+ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
+ try {
+ const responses = await qstash.batchJSON(
+ jobChunk.map((job) => buildReplayRequest(job, now)),
+ );
+
+ jobChunk.forEach((job, index) => {
+ const response = responses[index];
+
+ if (isPublishSuccess(response)) {
+ results.push({
+ id: job.id,
+ status: "published",
+ lastError: null,
+ messageId: response.messageId,
+ });
+ } else {
+ results.push({
+ id: job.id,
+ status: "failed",
+ lastError: "QStash batch publish did not return a messageId",
+ });
+ }
+ });
+ } catch (error) {
+ const lastError = (
+ error instanceof Error ? error.message : String(error)
+ ).slice(0, LAST_ERROR_MAX_LENGTH);
+
+ logger.error("jobs.publish_failed", {
+ jobCount: jobChunk.length,
+ errorMessage: lastError,
+ });
+
+ for (const job of jobChunk) {
+ results.push({
+ id: job.id,
+ status: "failed",
+ lastError,
+ });
+ }
+ }
+ }
+
+ await logger.flush();
+
+ return results;
+}
diff --git a/apps/web/lib/jobs/send-workflows.ts b/apps/web/lib/jobs/send-workflows.ts
new file mode 100644
index 0000000000..cde6de27e2
--- /dev/null
+++ b/apps/web/lib/jobs/send-workflows.ts
@@ -0,0 +1,121 @@
+import { logger, toErrorFields } from "@/lib/axiom/server";
+import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
+import { PublishRequest } from "@upstash/qstash";
+import { Client, TriggerOptions } from "@upstash/workflow";
+import { LAST_ERROR_MAX_LENGTH, QSTASH_BATCH_CHUNK_SIZE } from "./constants";
+import type { PersistableJob, PublishResult } from "./types";
+
+const workflowClient = new Client({
+ baseUrl: process.env.QSTASH_URL || "https://qstash-us-east-1.upstash.io",
+ token: process.env.QSTASH_TOKEN || "",
+ ...(process.env.VERCEL_ENV === "preview" && {
+ headers: {
+ "x-vercel-protection-bypass":
+ process.env.VERCEL_AUTOMATION_BYPASS_SECRET || "",
+ },
+ }),
+});
+
+const workflowPathMap = {
+ partnerApproved: "/api/workflows/partner-approved",
+ mergePartnerAccounts: "/api/workflows/merge-partner-accounts",
+ createPartnerCommission: "/api/workflows/create-partner-commission",
+} as const;
+
+export type WorkflowName = keyof typeof workflowPathMap;
+
+export type WorkflowOptions = Pick<
+ PublishRequest,
+ "label" | "deduplicationId" | "retries" | "flowControl"
+>;
+
+export function isWorkflowName(name: string): name is WorkflowName {
+ return Object.prototype.hasOwnProperty.call(workflowPathMap, name);
+}
+
+function isTriggerSuccess(
+ response: unknown,
+): response is { workflowRunId: string } {
+ return (
+ typeof response === "object" &&
+ response !== null &&
+ "workflowRunId" in response &&
+ typeof response.workflowRunId === "string"
+ );
+}
+
+function buildTriggerRequest(job: PersistableJob): TriggerOptions {
+ const name = job.name as WorkflowName;
+ const options = (job.options ?? {}) as WorkflowOptions;
+ const workflowPath = workflowPathMap[name];
+ const workflowKey = workflowPath.split("/").pop()!;
+
+ return {
+ url: `${APP_DOMAIN_WITH_NGROK}${workflowPath}`,
+ body: job.payload,
+ workflowRunId: options.deduplicationId ?? job.id,
+ retries: options.retries ?? 5,
+ flowControl: options.flowControl ?? {
+ key: workflowKey,
+ parallelism: 15,
+ },
+ ...(options.label && { label: options.label }),
+ };
+}
+
+/** Publish workflows to QStash Workflows (no DB writes). */
+export async function triggerWorkflows(
+ jobs: PersistableJob[],
+): Promise<PublishResult[]> {
+ if (jobs.length === 0) {
+ return [];
+ }
+
+ const results: PublishResult[] = [];
+
+ for (const jobChunk of chunk(jobs, QSTASH_BATCH_CHUNK_SIZE)) {
+ try {
+ const responses = await workflowClient.trigger(
+ jobChunk.map((job) => buildTriggerRequest(job)),
+ );
+
+ jobChunk.forEach((job, index) => {
+ const response = responses[index];
+
+ if (isTriggerSuccess(response)) {
+ results.push({
+ id: job.id,
+ status: "published",
+ lastError: null,
+ workflowRunId: response.workflowRunId,
+ });
+ } else {
+ results.push({
+ id: job.id,
+ status: "failed",
+ lastError: "Workflow trigger did not return a workflowRunId",
+ });
+ }
+ });
+ } catch (error) {
+ const lastError = serializeError(error).slice(0, LAST_ERROR_MAX_LENGTH);
+
+ logger.error("workflows.publish_failed", {
+ jobCount: jobChunk.length,
+ error: toErrorFields(error),
+ });
+
+ for (const job of jobChunk) {
+ results.push({
+ id: job.id,
+ status: "failed",
+ lastError,
+ });
+ }
+ }
+ }
+
+ await logger.flush();
+
+ return results;
+}
diff --git a/apps/web/lib/jobs/types.ts b/apps/web/lib/jobs/types.ts
new file mode 100644
index 0000000000..769d18d8ff
--- /dev/null
+++ b/apps/web/lib/jobs/types.ts
@@ -0,0 +1,15 @@
+export type PublishResult = {
+ id: string;
+ status: "published" | "failed";
+ lastError: string | null;
+ messageId?: string;
+ workflowRunId?: string;
+};
+
+export type PersistableJob = {
+ id: string;
+ name: string;
+ payload: unknown;
+ options?: unknown;
+ scheduledAt?: Date;
+};
diff --git a/apps/web/prisma/schema/job.prisma b/apps/web/prisma/schema/job.prisma
index 779b3b5859..3d62641121 100644
--- a/apps/web/prisma/schema/job.prisma
+++ b/apps/web/prisma/schema/job.prisma
@@ -1,21 +1,15 @@
-enum JobStatus {
- pending
- published
- failed
-}
-
+// Background jobs that could not be published to QStash (e.g. QStash outage).
+// The /api/cron/queue/retry cron republishes these and deletes them on success.
model Job {
- id String @id
+ id String @id
name String
payload Json
options Json? // dispatch options replayed verbatim: deduplicationId, retries, queue, flowControl, label
scheduledAt DateTime @default(now()) // when we should publish the job
- status JobStatus @default(pending)
- attempts Int @default(0)
- lastError String? @db.Text
- createdAt DateTime @default(now())
- updatedAt DateTime @updatedAt
+ attempts Int @default(0)
+ lastError String? @db.Text
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
- @@index([status, scheduledAt, attempts]) // retry cron: due pending/failed jobs
- @@index([status, createdAt]) // daily cleanup by status + age
+ @@index([scheduledAt, attempts]) // retry cron: due jobs under the attempt cap
}