Open source CapCut alternative.
TypeScript
feat(site): drain the email outbox one kind at a time from su
David Leeebf6889d6a7
diff --git a/site/src/app/api/su/email/route.ts b/site/src/app/api/su/email/route.tsindex b8051b44..f9a853f2 100644--- a/site/src/app/api/su/email/route.ts+++ b/site/src/app/api/su/email/route.ts@@ -3,7 +3,8 @@ import { z } from "zod";import { invalidResponse } from "@/lib/config/experimentList";import { notFoundResponse, withSuperUser } from "@/lib/donkey-api-auth";-import { outboxOverview, retryEmail, scheduleDrain } from "@/lib/email/outbox";+import { EMAIL_KIND_IDS } from "@/lib/email/kindIds";+import { drainKindNow, outboxOverview, retryEmail } from "@/lib/email/outbox";// Without a queue configured (local dev) a drain runs inline before the// response; give it room.@@ -12,16 +13,16 @@ export const maxDuration = 300;export const GET = withSuperUser(async () => NextResponse.json(await outboxOverview()));const actionSchema = z.union([- z.object({ action: z.literal("drain") }).strict(),+ z.object({ action: z.literal("drain"), kind: z.enum(EMAIL_KIND_IDS) }).strict(),z.object({ action: z.literal("retry"), id: z.string().min(1) }).strict(),]);-// Drain now, or put one failed row back in the queue.+// Drain one kind now, or put one failed row back in the queue.export const POST = withSuperUser(async (request) => {const parsed = actionSchema.safeParse(await request.json());if (!parsed.success) return invalidResponse(parsed.error.issues);if (parsed.data.action === "drain") {- await scheduleDrain(0);+ await drainKindNow(parsed.data.kind, request.donkey.userId);} else if (!(await retryEmail(parsed.data.id))) {return notFoundResponse();}diff --git a/site/src/app/su/SuHeader.tsx b/site/src/app/su/SuHeader.tsxindex 400738df..4983ddc3 100644--- a/site/src/app/su/SuHeader.tsx+++ b/site/src/app/su/SuHeader.tsx@@ -6,7 +6,6 @@ import type { ComponentType } from "react";import { RunAnalyticsButton } from "@/app/su/analytics/RunAnalyticsButton";import { NewPostButton } from "@/app/su/blog/NewPostButton";-import { DrainNowButton } from "@/app/su/jobs/email/DrainNowButton";import { suSurfaceAt, type SuPage } from "@/app/su/nav";import { useSuCrumbValue } from "@/app/su/SuCrumb";import {@@ -26,7 +25,6 @@ import { SidebarTrigger } from "@/components/ui/sidebar";const PAGE_ACTIONS: Partial<Record<SuPage["href"], ComponentType>> = {"/analytics/product": RunAnalyticsButton,"/blog": NewPostButton,- "/jobs/email": DrainNowButton,"/outreach": ScanOutreachButton,};diff --git a/site/src/app/su/jobs/email/DrainNowButton.tsx b/site/src/app/su/jobs/email/DrainNowButton.tsxdeleted file mode 100644index 805db16c..00000000--- a/site/src/app/su/jobs/email/DrainNowButton.tsx+++ /dev/null@@ -1,23 +0,0 @@-"use client";--import { Button } from "@/components/ui/button";-import { useEmailOutbox, useOutboxAction } from "@/queries/emailOutbox";--export function DrainNowButton() {- const outbox = useEmailOutbox();- const drain = useOutboxAction();- const pending = outbox.data?.drainPending ?? false;- return (- <div className="flex items-center gap-3">- {drain.isError ? <span className="text-sm text-destructive">Drain failed.</span> : null}- <Button- disabled={drain.isPending || pending}- onClick={() => drain.mutate({ action: "drain" })}- size="sm"- variant="outline"- >- {pending ? "Draining…" : "Drain now"}- </Button>- </div>- );-}diff --git a/site/src/app/su/jobs/email/page.tsx b/site/src/app/su/jobs/email/page.tsxindex 3e05b9e9..f52ce701 100644--- a/site/src/app/su/jobs/email/page.tsx+++ b/site/src/app/su/jobs/email/page.tsx@@ -95,6 +95,7 @@ function QuotaSection({ quota }: { quota: OutboxOverview["quota"] }) {}function KindsSection({ kinds }: { kinds: OutboxOverview["kinds"] }) {+ const action = useOutboxAction();const cell = "py-1.5 pr-3";return (<section className="space-y-2">@@ -108,7 +109,8 @@ function KindsSection({ kinds }: { kinds: OutboxOverview["kinds"] }) {<th className="py-1 pr-3 font-normal">Quota</th><th className="py-1 pr-3 text-right font-normal">Queued</th><th className="py-1 pr-3 text-right font-normal">Sent today</th>- <th className="py-1 text-right font-normal">Failed</th>+ <th className="py-1 pr-3 text-right font-normal">Failed</th>+ <th className="py-1" /></tr></thead><tbody>@@ -121,7 +123,22 @@ function KindsSection({ kinds }: { kinds: OutboxOverview["kinds"] }) {<td className={cn(cell, "text-muted-foreground")}>{k.quota}</td><td className={cn(cell, "text-right tabular-nums")}>{k.queued}</td><td className={cn(cell, "text-right tabular-nums")}>{k.sentToday}</td>- <td className={cn("py-1.5 text-right tabular-nums", k.failed > 0 && "text-destructive")}>{k.failed}</td>+ <td className={cn(cell, "text-right tabular-nums", k.failed > 0 && "text-destructive")}>{k.failed}</td>+ <td className="py-1 text-right">+ <span className="inline-flex items-center gap-2">+ {action.isError && action.variables?.action === "drain" && action.variables.kind === k.kind ? (+ <span className="text-xs text-destructive">Failed.</span>+ ) : null}+ <Button+ disabled={k.queued === 0 || k.draining || action.isPending}+ onClick={() => action.mutate({ action: "drain", kind: k.kind })}+ size="sm"+ variant="outline"+ >+ {k.draining ? "Draining…" : "Drain"}+ </Button>+ </span>+ </td></tr>))}</tbody>diff --git a/site/src/lib/email/outbox.ts b/site/src/lib/email/outbox.tsindex b9719de3..df7639aa 100644--- a/site/src/lib/email/outbox.ts+++ b/site/src/lib/email/outbox.ts@@ -112,6 +112,14 @@ export async function scheduleDrain(delaySeconds: number): Promise<void> {await ensureJob("email-drain", {}, "system", { delaySeconds });}+/** Starts a drainer for one kind now, on an operator's say-so. Its own job,+ * so a drainer held back by another kind's quota wait does not stand in+ * for it. */+export async function drainKindNow(kind: EmailKindId, createdBy: string): Promise<void> {+ const { enqueueJob } = await import("@/lib/jobs/queue");+ await enqueueJob("email-drain", { kind }, createdBy);+}+type DrainableRow = Awaited<ReturnType<typeof prisma.emailSend.findMany>>[number];async function sendRow(row: DrainableRow, now: Date): Promise<Outcome> {@@ -184,8 +192,9 @@ export type DrainResult = {/** Sends what the outbox holds, highest priority first, until the budget or* the quota runs out. A refused quota class is left alone for the rest of- * the run; the classes above it keep going. */-export async function drainOutbox(budgetMs: number): Promise<DrainResult> {+ * the run; the classes above it keep going. Given a kind, only that kind's+ * rows go. */+export async function drainOutbox(budgetMs: number, kind?: EmailKindId): Promise<DrainResult> {const startedAt = Date.now();const result: DrainResult = { sent: 0, failed: 0, skipped: 0, retried: 0, retryAfterSeconds: null, more: false };const refused = new Map<EmailSendKind, number>();@@ -196,13 +205,14 @@ export async function drainOutbox(budgetMs: number): Promise<DrainResult> {drain: for (;;) {const now = new Date();+ if (kind && refused.has(EMAIL_KINDS[kind].quota)) break;const rows = await prisma.emailSend.findMany({orderBy: [{ priority: "desc" }, { rank: "asc" }, { createdAt: "asc" }],take: PAGE,where: {AND: [claimable(now),- { kind: { notIn: refusedKinds() }, notBefore: { lte: now } },+ { kind: kind ?? { notIn: refusedKinds() }, notBefore: { lte: now } },// A paused promotion's rows wait where they are.{ OR: [{ promotionId: null }, { promotion: { status: "sending" } }] },],@@ -229,18 +239,19 @@ export async function drainOutbox(budgetMs: number): Promise<DrainResult> {await finishPromotions(touchedPromotions);const waits = [...refused.values()];- const backedOff = await nextBackedOffRow(new Date());+ const backedOff = await nextBackedOffRow(new Date(), kind);if (backedOff) waits.push(Math.max(1, Math.ceil((backedOff.getTime() - Date.now()) / 1000)));if (waits.length > 0) result.retryAfterSeconds = Math.min(...waits);return result;}// When the soonest row held back by a failed attempt comes due.-async function nextBackedOffRow(now: Date): Promise<Date | null> {+async function nextBackedOffRow(now: Date, kind?: EmailKindId): Promise<Date | null> {const row = await prisma.emailSend.findFirst({orderBy: { notBefore: "asc" },select: { notBefore: true },where: {+ kind,notBefore: { gt: now },state: "queued",OR: [{ promotionId: null }, { promotion: { status: "sending" } }],@@ -268,6 +279,9 @@ export type OutboxKindRow = {queued: number;sentToday: number;failed: number;+ // A drainer covering this kind is running or due now; one held back by a+ // quota wait is not.+ draining: boolean;};export type OutboxItem = {@@ -305,8 +319,6 @@ export type OutboxOverview = {campaigns: OutboxCampaign[];// What is waiting, then what failed, then what went most recently.items: OutboxItem[];- // A drainer is running or due now; one held back by a quota wait is not.- drainPending: boolean;};const ITEMS = 60;@@ -315,7 +327,7 @@ const ITEMS = 60;* rows worth a look. */export async function outboxOverview(now = new Date()): Promise<OutboxOverview> {const dayStart = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));- const [quota, priorities, queued, sentToday, failed, byCampaign, waiting, broken, recent, drain] = await Promise.all([+ const [quota, priorities, queued, sentToday, failed, byCampaign, waiting, broken, recent, drains] = await Promise.all([dailyEmailQuotaStatus(now),getGlobalSetting("emailPriorities"),prisma.emailSend.groupBy({ _count: true, by: ["kind"], where: { state: { in: ["queued", "sending"] } } }),@@ -340,14 +352,17 @@ export async function outboxOverview(now = new Date()): Promise<OutboxOverview>take: ITEMS,where: { state: { in: ["sent", "skipped"] } },}),- prisma.asyncJob.findFirst({- select: { id: true },+ prisma.asyncJob.findMany({+ select: { payload: true },where: {kind: "email-drain",OR: [{ state: "running" }, { notBefore: null, state: "queued" }, { notBefore: { lte: now }, state: "queued" }],},}),]);+ // A drainer without a kind covers every kind.+ const drainingKinds = drains.map((d) => (d.payload as { kind?: EmailKindId } | null)?.kind ?? null);+ const draining = (kind: EmailKindId) => drainingKinds.some((k) => k === null || k === kind);const countOf = (groups: { kind: string; _count: number }[], kind: string) =>groups.find((g) => g.kind === kind)?._count ?? 0;const kinds = EMAIL_KIND_IDS.map((kind) => ({@@ -357,6 +372,7 @@ export async function outboxOverview(now = new Date()): Promise<OutboxOverview>queued: countOf(queued, kind),sentToday: countOf(sentToday, kind),failed: countOf(failed, kind),+ draining: draining(kind),}));const campaignIds = [...new Set(byCampaign.map((g) => g.promotionId).filter((id): id is string => id !== null))];const names = await prisma.promotion.findMany({ select: { id: true, name: true }, where: { id: { in: campaignIds } } });@@ -388,7 +404,7 @@ export async function outboxOverview(now = new Date()): Promise<OutboxOverview>sentAt: row.sentAt?.toISOString() ?? null,createdAt: row.createdAt.toISOString(),}));- return { quota, kinds, campaigns, items, drainPending: drain !== null };+ return { quota, kinds, campaigns, items };}/** Puts a failed row back in the queue with a clean slate and sends for adiff --git a/site/src/lib/jobs/email-drain.ts b/site/src/lib/jobs/email-drain.tsindex adeffea8..c17ffd58 100644--- a/site/src/lib/jobs/email-drain.ts+++ b/site/src/lib/jobs/email-drain.ts@@ -1,5 +1,6 @@import { z } from "zod";+import { EMAIL_KIND_IDS, type EmailKindId } from "@/lib/email/kindIds";import { drainOutbox, type DrainResult } from "@/lib/email/outbox";import { defineJob, JobDeferred } from "@/lib/jobs/registry";@@ -9,12 +10,12 @@ const BUDGET_MS = 200_000;/** Runs the drainer inside a job: a run that ran out of time queues its* continuation, and one that ran out of quota comes back when the quota is- * expected to open. */-export async function drainAsJob(): Promise<DrainResult> {- const result = await drainOutbox(BUDGET_MS);+ * expected to open. A run for one kind continues for that kind. */+export async function drainAsJob(kind?: EmailKindId): Promise<DrainResult> {+ const result = await drainOutbox(BUDGET_MS, kind);if (result.more) {const { enqueueJob } = await import("@/lib/jobs/queue");- await enqueueJob("email-drain", {}, "system");+ await enqueueJob("email-drain", kind ? { kind } : {}, "system");return result;}if (result.retryAfterSeconds !== null) {@@ -23,4 +24,7 @@ export async function drainAsJob(): Promise<DrainResult> {return result;}-export const emailDrainJob = defineJob(z.object({}).strict(), async () => drainAsJob());+export const emailDrainJob = defineJob(+ z.object({ kind: z.enum(EMAIL_KIND_IDS).optional() }).strict(),+ async ({ kind }) => drainAsJob(kind),+);diff --git a/site/src/queries/emailOutbox.ts b/site/src/queries/emailOutbox.tsindex a2c9488f..dacebca1 100644--- a/site/src/queries/emailOutbox.ts+++ b/site/src/queries/emailOutbox.ts@@ -2,6 +2,7 @@import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";+import type { EmailKindId } from "@/lib/email/kindIds";import type { OutboxOverview } from "@/lib/email/outbox";import { apiFetch } from "@/queries/apiClient";@@ -19,12 +20,12 @@ export function useEmailOutbox() {});}-// Drain now, or put one failed row back in the queue; the response is the-// refreshed overview.+// Drain one kind now, or put one failed row back in the queue; the response+// is the refreshed overview.export function useOutboxAction() {const queryClient = useQueryClient();return useMutation({- mutationFn: (body: { action: "drain" } | { action: "retry"; id: string }) =>+ mutationFn: (body: { action: "drain"; kind: EmailKindId } | { action: "retry"; id: string }) =>apiFetch<OutboxOverview>("/api/su/email", { body: JSON.stringify(body), method: "POST" }),onSuccess: (data) => queryClient.setQueryData(emailOutboxQueryKey, data),});