Skip to content

Open source CapCut alternative.

TypeScript

fix(site): keep the email outbox moving

David Le308b335e2419
diff --git a/site/prisma/Jobs.prisma b/site/prisma/Jobs.prisma
index 567f1665..37e87a0b 100644
--- a/site/prisma/Jobs.prisma
+++ b/site/prisma/Jobs.prisma
@@ -9,6 +9,9 @@ model AsyncJob {
kind String
payload Json
state String @default("queued") // queued | running | done | error
+ // When a held-back job is due; the message carrying it is delayed to match,
+ // so a poll before then republishes nothing.
+ notBefore DateTime?
result Json?
error String?
createdBy String
diff --git a/site/src/lib/email/daily-send-limit.ts b/site/src/lib/email/daily-send-limit.ts
index 4cba6ef0..7ce12817 100644
--- a/site/src/lib/email/daily-send-limit.ts
+++ b/site/src/lib/email/daily-send-limit.ts
@@ -99,13 +99,14 @@ async function releaseSlot(day: string): Promise<void> {
});
}
-/** Runs one provider send inside the day's quota. The slot is kept only when
- * the provider accepted the email: a thrown error or a resolved `{ error }`
- * gives it back. */
+/** Runs one provider send inside the day's quota, with the slot held before
+ * the callback starts so what it builds goes out at once. The slot is kept
+ * only when the provider accepted the email: a thrown error, a resolved
+ * `{ error }`, or null (nothing to send) gives it back. */
export async function withDailyEmailQuota<T extends { error: unknown }>(
kind: EmailSendKind,
- send: () => Promise<T>,
-): Promise<T> {
+ send: () => Promise<T | null>,
+): Promise<T | null> {
const now = new Date();
const day = utcDayOf(now);
await reserveSlot(day, kind, now);
@@ -116,13 +117,13 @@ export async function withDailyEmailQuota<T extends { error: unknown }>(
error: releaseError instanceof Error ? releaseError.message : String(releaseError),
});
});
- let result: T;
+ let result: T | null;
try {
result = await send();
} catch (error) {
await release();
throw error;
}
- if (result.error) await release();
+ if (result === null || result.error) await release();
return result;
}
diff --git a/site/src/lib/email/outbox.ts b/site/src/lib/email/outbox.ts
index c642f4a4..f9ca1bf3 100644
--- a/site/src/lib/email/outbox.ts
+++ b/site/src/lib/email/outbox.ts
@@ -41,8 +41,12 @@ export type QueueEmailInput = {
export type DeliverResult = { id: string; state: "sent" | "queued" | "failed" | "skipped"; error: string | null };
type Outcome =
- | { outcome: "sent" | "retried" | "failed" | "skipped" | "busy" }
- | { outcome: "refused"; retryAfterSeconds: number };
+ | { outcome: "sent" | "failed" | "skipped" | "busy" }
+ | { outcome: "refused" | "retried"; retryAfterSeconds: number };
+
+// A provider blip backs the row off, further each time, so a bad hour does
+// not burn its attempts.
+const backoffSeconds = (attempts: number) => attempts * 5 * 60;
function claimable(now: Date): Prisma.EmailSendWhereInput {
return {
@@ -75,12 +79,13 @@ async function rowsOf(inputs: QueueEmailInput[]) {
}));
}
-/** Queues many emails at once; rows already queued under the same key are
- * left as they are. Returns how many were new. */
+/** Queues many emails at once and sends for a drainer; rows already queued
+ * under the same key are left as they are, and still get the drainer, so a
+ * resumed promotion moves. Returns how many were new. */
export async function queueEmails(inputs: QueueEmailInput[]): Promise<number> {
if (inputs.length === 0) return 0;
const { count } = await prisma.emailSend.createMany({ data: await rowsOf(inputs), skipDuplicates: true });
- if (count > 0) await scheduleDrain(0);
+ await scheduleDrain(0);
return count;
}
@@ -92,24 +97,19 @@ export async function deliverEmail(input: QueueEmailInput): Promise<DeliverResul
const row = await prisma.emailSend.findUniqueOrThrow({ where: { idempotencyKey: input.idempotencyKey } });
if (row.state === "queued") {
const outcome = await sendRow(row, new Date());
- if (outcome.outcome === "refused") await scheduleDrain(outcome.retryAfterSeconds);
+ if (outcome.outcome === "refused" || outcome.outcome === "retried") await scheduleDrain(outcome.retryAfterSeconds);
}
const settled = await prisma.emailSend.findUniqueOrThrow({ select: { error: true, id: true, state: true }, where: { id: row.id } });
const state = settled.state === "sending" ? "queued" : (settled.state as DeliverResult["state"]);
return { error: settled.error, id: settled.id, state };
}
-/** Makes sure a drainer is coming, no sooner than the delay. One pending
- * drain job serves every caller. */
+/** Makes sure a drainer is coming no later than the delay. One pending
+ * drain job serves every caller; one held back for later is pulled forward. */
export async function scheduleDrain(delaySeconds: number): Promise<void> {
- const pending = await prisma.asyncJob.findFirst({
- select: { id: true },
- where: { kind: "email-drain", state: { in: ["queued", "running"] } },
- });
- if (pending) return;
// Imported here because the job registry imports this module.
- const { enqueueJob } = await import("@/lib/jobs/queue");
- await enqueueJob("email-drain", {}, "system", { delaySeconds });
+ const { ensureJob } = await import("@/lib/jobs/queue");
+ await ensureJob("email-drain", {}, "system", { delaySeconds });
}
type DrainableRow = Awaited<ReturnType<typeof prisma.emailSend.findMany>>[number];
@@ -139,14 +139,16 @@ async function sendRow(row: DrainableRow, now: Date): Promise<Outcome> {
try {
if (!isResendConfigured()) throw new ResendNotConfiguredError();
- const message = await kind.build(payload, outboxRow);
- if (!message) {
+ // The slot is held before the message is built, so anything the build
+ // writes down (a claim window, say) dates from the send itself.
+ const result = await withDailyEmailQuota(kind.quota, async () => {
+ const message = await kind.build(payload, outboxRow);
+ return message ? getResend().emails.send(message, { idempotencyKey: row.idempotencyKey }) : null;
+ });
+ if (result === null) {
await settle({ attempts, error: null, state: "skipped" });
return { outcome: "skipped" };
}
- const result = await withDailyEmailQuota(kind.quota, () =>
- getResend().emails.send(message, { idempotencyKey: row.idempotencyKey }),
- );
if (result.error) throw new Error(`Resend send failed: ${result.error.name}: ${result.error.message}`);
await settle({ attempts, error: null, sentAt: new Date(), state: "sent" });
await kind.afterSend?.(payload, outboxRow);
@@ -157,15 +159,14 @@ async function sendRow(row: DrainableRow, now: Date): Promise<Outcome> {
return { outcome: "refused", retryAfterSeconds: error.retryAfterSeconds };
}
if (isPermanent(error) || attempts >= MAX_ATTEMPTS) return giveUp(error, payload);
- // A provider blip: back off, further each time, so a bad hour does not
- // burn the row's attempts.
+ const retryAfterSeconds = backoffSeconds(attempts);
await settle({
attempts,
error: messageOf(error),
- notBefore: new Date(now.getTime() + attempts * 5 * 60_000),
+ notBefore: new Date(now.getTime() + retryAfterSeconds * 1000),
state: "queued",
});
- return { outcome: "retried" };
+ return { outcome: "retried", retryAfterSeconds };
}
}
@@ -174,7 +175,8 @@ export type DrainResult = {
failed: number;
skipped: number;
retried: number;
- // Seconds until a refused quota is expected to open; null when nothing was refused.
+ // Seconds until the next thing worth coming back for: a refused quota
+ // expected to open, or a backed-off row coming due. Null when nothing waits.
retryAfterSeconds: number | null;
// Whether the budget ran out with rows still owed.
more: boolean;
@@ -226,10 +228,27 @@ export async function drainOutbox(budgetMs: number): Promise<DrainResult> {
}
await finishPromotions(touchedPromotions);
- if (refused.size > 0) result.retryAfterSeconds = Math.min(...refused.values());
+ const waits = [...refused.values()];
+ const backedOff = await nextBackedOffRow(new Date());
+ if (backedOff) waits.push(Math.max(1, Math.ceil((backedOff.getTime() - Date.now()) / 1000)));
+ if (waits.length > 0) result.retryAfterSeconds = Math.min(...waits);
return result;
}
+// When the soonest row held back by a failed attempt comes due.
+async function nextBackedOffRow(now: Date): Promise<Date | null> {
+ const row = await prisma.emailSend.findFirst({
+ orderBy: { notBefore: "asc" },
+ select: { notBefore: true },
+ where: {
+ notBefore: { gt: now },
+ state: "queued",
+ OR: [{ promotionId: null }, { promotion: { status: "sending" } }],
+ },
+ });
+ return row?.notBefore ?? null;
+}
+
// A promotion with nothing left to send is sent.
async function finishPromotions(promotionIds: Set<string>): Promise<void> {
for (const promotionId of promotionIds) {
@@ -273,6 +292,7 @@ export type OutboxOverview = {
kinds: OutboxKindRow[];
// What is waiting, then what failed, then what went most recently.
items: OutboxItem[];
+ // A drainer is running or due now; one held back by a quota wait is not.
drainPending: boolean;
};
@@ -306,7 +326,13 @@ export async function outboxOverview(now = new Date()): Promise<OutboxOverview>
take: ITEMS,
where: { state: { in: ["sent", "skipped"] } },
}),
- prisma.asyncJob.findFirst({ select: { id: true }, where: { kind: "email-drain", state: { in: ["queued", "running"] } } }),
+ prisma.asyncJob.findFirst({
+ select: { id: true },
+ where: {
+ kind: "email-drain",
+ OR: [{ state: "running" }, { notBefore: null, state: "queued" }, { notBefore: { lte: now }, state: "queued" }],
+ },
+ }),
]);
const countOf = (groups: { kind: string; _count: number }[], kind: string) =>
groups.find((g) => g.kind === kind)?._count ?? 0;
diff --git a/site/src/lib/jobs/queue.ts b/site/src/lib/jobs/queue.ts
index a721228e..75226a2f 100644
--- a/site/src/lib/jobs/queue.ts
+++ b/site/src/lib/jobs/queue.ts
@@ -81,18 +81,54 @@ export async function enqueueJob(
createdBy: string,
options: { delaySeconds?: number } = {},
): Promise<{ jobId: string }> {
- const job = await prisma.asyncJob.create({ data: { kind, payload, createdBy } });
- const queued = await publishJob(job.id, options.delaySeconds);
+ const delaySeconds = options.delaySeconds ?? 0;
+ const job = await prisma.asyncJob.create({
+ data: { createdBy, kind, notBefore: delaySeconds > 0 ? dueAt(delaySeconds) : null, payload },
+ });
+ const queued = await publishJob(job.id, delaySeconds);
if (!queued) await executeJob(job.id);
return { jobId: job.id };
}
+function dueAt(delaySeconds: number): Date {
+ return new Date(Date.now() + delaySeconds * 1000);
+}
+
+/** One job of this kind on its way, no later than the delay. A pending job
+ * that is due sooner is left alone; one held back for later is pulled
+ * forward; none means a new one. Kinds whose work is shared, such as the
+ * email drainer, are asked for this way so every caller feeds one run. */
+export async function ensureJob(
+ kind: string,
+ payload: Prisma.InputJsonValue,
+ createdBy: string,
+ options: { delaySeconds?: number } = {},
+): Promise<{ jobId: string }> {
+ const delaySeconds = options.delaySeconds ?? 0;
+ const pending = await prisma.asyncJob.findFirst({
+ orderBy: { createdAt: "asc" },
+ select: { id: true, notBefore: true, state: true },
+ where: { kind, state: { in: ["queued", "running"] } },
+ });
+ if (!pending) return enqueueJob(kind, payload, createdBy, { delaySeconds });
+ const due = dueAt(delaySeconds);
+ if (pending.state === "queued" && pending.notBefore && pending.notBefore > due) {
+ // The earlier message still arrives on its own schedule; by then the row
+ // is settled and the claim turns that delivery away.
+ await prisma.asyncJob.update({ data: { notBefore: delaySeconds > 0 ? due : null }, where: { id: pending.id } });
+ await publishJob(pending.id, delaySeconds);
+ }
+ return { jobId: pending.id };
+}
+
/** Self-heal one polled row: a stale queued job re-publishes its message, and
* a stale running job (crashed execution) becomes an error instead of spinning
- * the poller forever. Returns the row, refreshed when it changed. */
+ * the poller forever. A job held back until later is left to its delayed
+ * message. Returns the row, refreshed when it changed. */
export async function healJob(job: JobRow): Promise<JobRow> {
const idleMs = Date.now() - job.updatedAt.getTime();
- if (job.state === "queued" && idleMs > STALE_QUEUED_MS) {
+ const held = job.notBefore !== null && job.notBefore.getTime() > Date.now();
+ if (job.state === "queued" && idleMs > STALE_QUEUED_MS && !held) {
await publishJob(job.id);
} else if (job.state === "running" && idleMs > STALE_RUNNING_MS) {
await prisma.asyncJob.updateMany({
@@ -162,7 +198,10 @@ export async function executeJob(jobId: string): Promise<Response> {
} catch (e) {
if (e instanceof JobDeferred) {
await prisma.asyncJob
- .updateMany({ where: { id: jobId, state: "running" }, data: { state: "queued" } })
+ .updateMany({
+ data: { notBefore: dueAt(e.retryAfterSeconds), state: "queued" },
+ where: { id: jobId, state: "running" },
+ })
.catch(() => {});
if (await publishJob(jobId, e.retryAfterSeconds)) return NextResponse.json({ ok: true });
return NextResponse.json(