Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Share outbox persist so deferred jobs use the same retry accounting.

Kiran Kf69b8bab7df5
diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.ts
index 84b8191678..f8cf8085b3 100644
--- a/apps/web/lib/jobs/index.ts
+++ b/apps/web/lib/jobs/index.ts
@@ -1,11 +1,9 @@
-import { createId } from "@/lib/api/create-id";
import { logger, toErrorFields } from "@/lib/axiom/server";
import { qstash } from "@/lib/cron";
-import { prisma } from "@/lib/prisma";
import { chunk } from "@dub/utils";
-import { Prisma } from "@prisma/client";
import * as z from "zod/v4";
import { QSTASH_BATCH_CHUNK_SIZE } from "./constants";
+import { persistBackgroundJobs } from "./outbox";
import {
buildJobLabel,
buildQStashJobRequest,
@@ -53,44 +51,6 @@ async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {
throw new Error("Failed to publish to QStash.");
}
-// Persist jobs that could not be published to QStash. The
-// /api/cron/queue/retry cron republishes them and deletes the rows on success.
-async function persistBackgroundJobs(inputs: DispatchJobInput[]) {
- const jobs = inputs.map(({ name, payload, options }) => {
- let scheduledAt = new Date();
-
- if (options?.notBefore) {
- scheduledAt = new Date(options.notBefore * 1000);
- } else if (typeof options?.delay === "number") {
- scheduledAt = new Date(Date.now() + options.delay * 1000);
- }
-
- const replayOptions = {
- ...(options?.deduplicationId && {
- deduplicationId: options.deduplicationId,
- }),
- ...(options?.retries !== undefined && { retries: options.retries }),
- ...(options?.queue && { queue: options.queue }),
- ...(options?.flowControl && { flowControl: options.flowControl }),
- ...(options?.label && { label: options.label }),
- };
-
- return {
- id: createId({ prefix: "job_" }),
- name,
- payload: payload as Prisma.InputJsonValue,
- options: replayOptions as Prisma.InputJsonValue,
- scheduledAt,
- };
- });
-
- await prisma.job.createMany({
- data: jobs,
- });
-
- return jobs;
-}
-
async function deferJobs(inputs: DispatchJobInput[]) {
const backgroundJobs = await persistBackgroundJobs(inputs);
diff --git a/apps/web/lib/jobs/outbox.ts b/apps/web/lib/jobs/outbox.ts
index 4c008eb7b8..2461e0a693 100644
--- a/apps/web/lib/jobs/outbox.ts
+++ b/apps/web/lib/jobs/outbox.ts
@@ -1,8 +1,9 @@
+import { createId } from "@/lib/api/create-id";
import { logger, toErrorFields } from "@/lib/axiom/server";
import { Job, Prisma } from "@prisma/client";
import { prisma } from "../prisma";
import { MAX_JOB_ATTEMPTS, MAX_JOBS_PER_BATCH } from "./constants";
-import { isDefineJobName, sendJobs } from "./send-jobs";
+import { isDefineJobName, sendJobs, type DispatchJobInput } from "./send-jobs";
import { isWorkflowName, triggerWorkflows } from "./send-workflows";
import type { PersistableJob, PublishResult } from "./types";
@@ -22,6 +23,25 @@ const transports: JobTransport[] = [
},
];
+// First failed publish counts as attempt 1 so retry budget matches MAX_JOB_ATTEMPTS.
+function toJobCreateInput({
+ job,
+ lastError,
+}: {
+ job: PersistableJob;
+ lastError?: string | null;
+}): Prisma.JobCreateManyInput {
+ return {
+ id: job.id,
+ name: job.name,
+ payload: job.payload as Prisma.InputJsonValue,
+ options: (job.options ?? {}) as Prisma.InputJsonValue,
+ scheduledAt: job.scheduledAt ?? new Date(),
+ lastError,
+ attempts: 1,
+ };
+}
+
// Persist jobs that failed to publish. Swallow DB errors (log only)
export async function persistFailedJobs({
jobs,
@@ -42,19 +62,12 @@ export async function persistFailedJobs({
const data = jobs
.filter((job) => failedById.has(job.id))
- .map((job) => {
- const failed = failedById.get(job.id)!;
-
- return {
- id: job.id,
- name: job.name,
- payload: job.payload as Prisma.InputJsonValue,
- options: (job.options ?? {}) as Prisma.InputJsonValue,
- scheduledAt: job.scheduledAt ?? new Date(),
- lastError: failed.lastError,
- attempts: 1,
- };
- });
+ .map((job) =>
+ toJobCreateInput({
+ job,
+ lastError: failedById.get(job.id)!.lastError,
+ }),
+ );
try {
await prisma.job.createMany({
@@ -70,6 +83,44 @@ export async function persistFailedJobs({
}
}
+// Persist jobs that could not be published to QStash.
+// /api/cron/queue/retry cron republishes them and deletes the rows on success.
+export async function persistBackgroundJobs(inputs: DispatchJobInput[]) {
+ const jobs: PersistableJob[] = inputs.map(({ name, payload, options }) => {
+ let scheduledAt = new Date();
+
+ if (options?.notBefore) {
+ scheduledAt = new Date(options.notBefore * 1000);
+ } else if (typeof options?.delay === "number") {
+ scheduledAt = new Date(Date.now() + options.delay * 1000);
+ }
+
+ const replayOptions = {
+ ...(options?.deduplicationId && {
+ deduplicationId: options.deduplicationId,
+ }),
+ ...(options?.retries !== undefined && { retries: options.retries }),
+ ...(options?.queue && { queue: options.queue }),
+ ...(options?.flowControl && { flowControl: options.flowControl }),
+ ...(options?.label && { label: options.label }),
+ };
+
+ return {
+ id: createId({ prefix: "job_" }),
+ name,
+ payload,
+ options: replayOptions,
+ scheduledAt,
+ };
+ });
+
+ await prisma.job.createMany({
+ data: jobs.map((job) => toJobCreateInput({ job })),
+ });
+
+ return jobs;
+}
+
// Delete successfully republished rows; bump attempts on failures.
export async function settlePublishResults({
results,