Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Tighten job helpers and fail unknown kinds on retry.

Kiran K0e7a7a51bff9
diff --git a/apps/web/lib/jobs/index.ts b/apps/web/lib/jobs/index.ts
index 94650c516b..84b8191678 100644
--- a/apps/web/lib/jobs/index.ts
+++ b/apps/web/lib/jobs/index.ts
@@ -11,6 +11,7 @@ import {
buildQStashJobRequest,
isPublishSuccess,
jobNameSchema,
+ type DispatchJobInput,
type JobDispatchOptions,
} from "./send-jobs";
@@ -31,12 +32,6 @@ type DispatchBatchResult = {
results: DispatchResult[];
};
-interface DispatchJobInput {
- name: string;
- payload: unknown;
- options?: JobDispatchOptions;
-}
-
const QSTASH_PUBLISH_MAX_RETRIES = 3;
async function withQStashRetry<T>(fn: () => Promise<T>): Promise<T> {
diff --git a/apps/web/lib/jobs/outbox.ts b/apps/web/lib/jobs/outbox.ts
index ee8859873f..4c008eb7b8 100644
--- a/apps/web/lib/jobs/outbox.ts
+++ b/apps/web/lib/jobs/outbox.ts
@@ -14,7 +14,7 @@ type JobTransport = {
const transports: JobTransport[] = [
{
matches: isWorkflowName,
- send: (jobs) => triggerWorkflows(jobs),
+ send: triggerWorkflows,
},
{
matches: isDefineJobName,
@@ -22,7 +22,7 @@ const transports: JobTransport[] = [
},
];
-/** Persist jobs that failed to publish. Swallow DB errors (log only). */
+// Persist jobs that failed to publish. Swallow DB errors (log only)
export async function persistFailedJobs({
jobs,
failedResults,
@@ -70,7 +70,7 @@ export async function persistFailedJobs({
}
}
-/** Delete successfully republished rows; bump attempts on failures. */
+// Delete successfully republished rows; bump attempts on failures.
export async function settlePublishResults({
results,
jobs,
@@ -128,10 +128,10 @@ export async function settlePublishResults({
),
);
+ const failedIds = new Set(failedResults.map((result) => result.id));
+
const exhaustedJobs = jobs.filter(
- (job) =>
- failedResults.some((result) => result.id === job.id) &&
- job.attempts + 1 >= MAX_JOB_ATTEMPTS,
+ (job) => failedIds.has(job.id) && job.attempts + 1 >= MAX_JOB_ATTEMPTS,
);
if (exhaustedJobs.length > 0) {
@@ -147,7 +147,7 @@ export async function settlePublishResults({
};
}
-/** Select due rows once (indexed), send via matching transport, then settle. */
+// Select due rows once (indexed), send via matching transport, then settle.
export async function publishPendingJobs(): Promise<{
attempted: number;
published: number;
@@ -169,27 +169,39 @@ export async function publishPendingJobs(): Promise<{
});
if (jobs.length === 0) {
- return { attempted: 0, published: 0, failed: 0 };
+ return {
+ attempted: 0,
+ published: 0,
+ failed: 0,
+ };
}
const matched = new Set<string>();
+ const pending: Promise<PublishResult[]>[] = [];
- const results = (
- await Promise.all(
- transports.map((transport) => {
- const batch = jobs.filter((job) => transport.matches(job.name));
- for (const job of batch) {
- matched.add(job.id);
- }
- return transport.send(batch);
- }),
- )
- ).flat();
+ for (const transport of transports) {
+ const batch = jobs.filter((job) => transport.matches(job.name));
+
+ for (const job of batch) {
+ matched.add(job.id);
+ }
+
+ pending.push(transport.send(batch));
+ }
+
+ const results = (await Promise.all(pending)).flat();
for (const job of jobs) {
- if (!matched.has(job.id)) {
- logger.error("jobs.unknown_kind", { id: job.id, name: job.name });
+ if (matched.has(job.id)) {
+ continue;
}
+
+ logger.error("jobs.unknown_kind", { id: job.id, name: job.name });
+ results.push({
+ id: job.id,
+ status: "failed",
+ lastError: `Unknown job kind: ${job.name}`,
+ });
}
if (matched.size < jobs.length) {
diff --git a/apps/web/lib/jobs/send-jobs.ts b/apps/web/lib/jobs/send-jobs.ts
index 993e70b525..318e5d4478 100644
--- a/apps/web/lib/jobs/send-jobs.ts
+++ b/apps/web/lib/jobs/send-jobs.ts
@@ -1,6 +1,6 @@
-import { logger } from "@/lib/axiom/server";
+import { logger, toErrorFields } from "@/lib/axiom/server";
import { qstash } from "@/lib/cron";
-import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";
+import { APP_DOMAIN_WITH_NGROK, chunk, serializeError } from "@dub/utils";
import { Job, Prisma } from "@prisma/client";
import { PublishRequest } from "@upstash/qstash";
import * as z from "zod/v4";
@@ -24,7 +24,7 @@ type JobReplayOptions = Pick<
"deduplicationId" | "retries" | "queue" | "flowControl" | "label"
>;
-type DispatchJobInput = {
+export type DispatchJobInput = {
name: string;
payload: unknown;
options?: JobDispatchOptions;
@@ -51,7 +51,7 @@ export function isDefineJobName(name: string) {
return jobNameSchema.safeParse(name).success;
}
-export function getJobsEndpointUrl(name: string) {
+function getJobsEndpointUrl(name: string) {
return `${JOBS_ENDPOINT_URL}/${name}`;
}
@@ -59,10 +59,7 @@ export function buildJobLabel(name: string, label?: string) {
return label ? `${label},${name}` : name;
}
-export function buildJobDeduplicationId(
- name: string,
- deduplicationId?: string,
-) {
+function buildJobDeduplicationId(name: string, deduplicationId?: string) {
if (!deduplicationId) {
return undefined;
}
@@ -106,13 +103,13 @@ export function buildQStashJobRequest(
};
}
-export function buildReplayRequest(
+function buildReplayRequest(
job: {
name: string;
payload: Prisma.JsonValue;
options: Prisma.JsonValue | null;
createdAt: Date;
- scheduledAt: Date | null;
+ scheduledAt: Date;
},
now: Date = new Date(),
) {
@@ -181,13 +178,11 @@ export async function sendJobs(jobs: Job[]): Promise<PublishResult[]> {
}
});
} catch (error) {
- const lastError = (
- error instanceof Error ? error.message : String(error)
- ).slice(0, LAST_ERROR_MAX_LENGTH);
+ const lastError = serializeError(error).slice(0, LAST_ERROR_MAX_LENGTH);
logger.error("jobs.publish_failed", {
jobCount: jobChunk.length,
- errorMessage: lastError,
+ error: toErrorFields(error),
});
for (const job of jobChunk) {