The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Delete published jobs older than 24 hours so the outbox does not grow unbounded.
Kiran K66233f6dd3f4
diff --git a/apps/web/app/(ee)/api/cron/queue/cleanup/route.ts b/apps/web/app/(ee)/api/cron/queue/cleanup/route.tsnew file mode 100644index 0000000000..921710da79--- /dev/null+++ b/apps/web/app/(ee)/api/cron/queue/cleanup/route.ts@@ -0,0 +1,21 @@+import { withCron } from "@/lib/cron/with-cron";+import { cleanupPublishedJobs } from "@/lib/jobs/cleanup-jobs";+import { logAndRespond } from "../../utils";++export const dynamic = "force-dynamic";+export const maxDuration = 600;++// Deletes published Job rows older than 24 hours so the outbox does not grow unbounded.+// Runs once every day at 02:00:00 AM UTC (0 2 * * *)+// GET /api/cron/queue/cleanup+export const GET = withCron(async () => {+ const { deletedCount } = await cleanupPublishedJobs();++ if (deletedCount === 0) {+ return logAndRespond("No published jobs older than 24 hours to delete.");+ }++ return logAndRespond(+ `Deleted ${deletedCount} published jobs older than 24 hours.`,+ );+});diff --git a/apps/web/lib/jobs/cleanup-jobs.ts b/apps/web/lib/jobs/cleanup-jobs.tsnew file mode 100644index 0000000000..2171da8e89--- /dev/null+++ b/apps/web/lib/jobs/cleanup-jobs.ts@@ -0,0 +1,52 @@+import { prisma } from "@/lib/prisma";+import { JobStatus } from "@prisma/client";+import { subHours } from "date-fns";++const MAX_JOBS_PER_BATCH = 1000;+const RETENTION_HOURS = 24;++export async function cleanupPublishedJobs(): Promise<{+ deletedCount: number;+}> {+ const cutoff = subHours(new Date(), RETENTION_HOURS);+ let deletedCount = 0;++ while (true) {+ const jobs = await prisma.job.findMany({+ where: {+ status: JobStatus.published,+ createdAt: {+ lt: cutoff,+ },+ },+ select: {+ id: true,+ },+ take: MAX_JOBS_PER_BATCH,+ });++ if (jobs.length === 0) {+ break;+ }++ const { count } = await prisma.job.deleteMany({+ where: {+ id: {+ in: jobs.map((job) => job.id),+ },+ },+ });++ deletedCount += count;++ if (jobs.length < MAX_JOBS_PER_BATCH) {+ break;+ }++ await new Promise((resolve) => setTimeout(resolve, 500));+ }++ return {+ deletedCount,+ };+}diff --git a/apps/web/lib/jobs/publish-jobs.ts b/apps/web/lib/jobs/publish-jobs.tsindex b006c44993..eb4e842301 100644--- a/apps/web/lib/jobs/publish-jobs.ts+++ b/apps/web/lib/jobs/publish-jobs.ts@@ -1,14 +1,14 @@import { logger } from "@/lib/axiom/server";import { pluck } from "@dub/utils";-import { JobStatus, Prisma } from "@prisma/client";+import { Job, JobStatus, Prisma } from "@prisma/client";import { createId } from "../api/create-id";import { prisma } from "../prisma";import { QStashJobPublisher } from "./publishers/qstash-job-publisher";import { QStashWorkflowPublisher } from "./publishers/qstash-workflow-publisher";-import type { QStashJobOptions } from "./publishers/types";+import type { PublishSendResult, QStashJobOptions } from "./publishers/types";const MAX_ATTEMPTS = 10;-const BATCH_SIZE = 100;+const MAX_JOBS_PER_BATCH = 100;type DispatchJobInput = {name: string;@@ -99,7 +99,7 @@ export async function publishPendingJobs(orderBy: {createdAt: "asc" as const,},- take: BATCH_SIZE,+ take: MAX_JOBS_PER_BATCH,}),});@@ -119,66 +119,86 @@ export async function publishPendingJobs(])).flat();- const publishedIds = results- .filter((result) => result.status === JobStatus.published)- .map((result) => result.id);+ const publishedResults = results.filter(+ (result) => result.status === JobStatus.published,+ );const failedResults = results.filter((result) => result.status === JobStatus.failed,);- // Job has been published successfully- if (publishedIds.length > 0) {- await prisma.job.updateMany({- where: {- id: {- in: publishedIds,- },- },- data: {- status: JobStatus.published,+ await markAsPublished({ publishedResults });+ await markAsFailed({ failedResults, jobs });++ return {+ attempted: jobs.length,+ published: publishedResults.length,+ failed: failedResults.length,+ };+}++async function markAsPublished({+ publishedResults,+}: {+ publishedResults: PublishSendResult[];+}) {+ if (publishedResults.length === 0) {+ return;+ }++ await prisma.job.updateMany({+ where: {+ id: {+ in: publishedResults.map((result) => result.id),},- });+ },+ data: {+ status: JobStatus.published,+ },+ });+}++async function markAsFailed({+ failedResults,+ jobs,+}: {+ failedResults: PublishSendResult[];+ jobs: Pick<Job, "id" | "name" | "attempts">[];+}) {+ if (failedResults.length === 0) {+ return;}- // Job has failed- if (failedResults.length > 0) {- await Promise.all(- failedResults.map((result) =>- prisma.job.update({- where: {- id: result.id,- },- data: {- status: JobStatus.failed,- lastError: result.lastError,- attempts: {- increment: 1,- },+ await Promise.all(+ failedResults.map((result) =>+ prisma.job.update({+ where: {+ id: result.id,+ },+ data: {+ status: JobStatus.failed,+ lastError: result.lastError,+ attempts: {+ increment: 1,},- }),- ),- );-- // Jobs that just ran out of attempts are excluded from future runs by the- // MAX_ATTEMPTS filter and need manual intervention- const exhaustedJobs = jobs.filter(- (job) =>- failedResults.some((result) => result.id === job.id) &&- job.attempts + 1 >= MAX_ATTEMPTS,- );-- if (exhaustedJobs.length > 0) {- logger.error("jobs.retry_exhausted", {- jobs: exhaustedJobs.map(({ id, name }) => ({ id, name })),- });- await logger.flush();+ },+ }),+ ),+ );++ // Jobs that just ran out of attempts are excluded from future runs by the+ // MAX_ATTEMPTS filter and need manual intervention+ const exhaustedJobs = jobs.filter(+ (job) =>+ failedResults.some((result) => result.id === job.id) &&+ job.attempts + 1 >= MAX_ATTEMPTS,+ );++ if (exhaustedJobs.length > 0) {+ for (const { id, name } of exhaustedJobs) {+ logger.error("jobs.retry_exhausted", { id, name });}- }- return {- attempted: jobs.length,- published: publishedIds.length,- failed: failedResults.length,- };+ await logger.flush();+ }}diff --git a/apps/web/vercel.json b/apps/web/vercel.jsonindex e59dfb5eda..7bbfb94172 100644--- a/apps/web/vercel.json+++ b/apps/web/vercel.json@@ -92,6 +92,10 @@"path": "/api/cron/queue/retry","schedule": "* * * * *"},+ {+ "path": "/api/cron/queue/cleanup",+ "schedule": "0 2 * * *"+ },{"path": "/api/cron/campaigns/queue-scheduled","schedule": "* * * * *"