Open source CapCut alternative.
TypeScript
feat(site): send every email through a prioritized outbox
David Le101f8142b359
diff --git a/site/prisma/Auth.prisma b/site/prisma/Auth.prismaindex 968bab41..49bd204e 100644--- a/site/prisma/Auth.prisma+++ b/site/prisma/Auth.prisma@@ -28,7 +28,7 @@ model User {onboarding UserOnboarding?emailSettings UserEmailSettings?outreach UserOutreach[]- promotionsReceived PromotionRecipient[]+ emailSends EmailSend[]experimentAssignments ExperimentAssignment[]experimentHoldout ExperimentHoldout?diff --git a/site/prisma/Marketing.prisma b/site/prisma/Marketing.prismaindex 18c22700..657d305f 100644--- a/site/prisma/Marketing.prisma+++ b/site/prisma/Marketing.prisma@@ -46,6 +46,7 @@ model UserOutreach {// dialog at the time — a template is a starting point, not a contract.model OutreachTemplate {+ promotion Json?id String @id @default(cuid())// What the send dialog calls it. Unique, so saving under an existing name// updates that template instead of leaving two the operator can't tell apart.@@ -69,6 +70,7 @@ model OutreachTemplate {// segment into recipient rows once and hands them to the promotion-send job.model Promotion {+ creditOffer Json?id String @id @default(cuid())name Stringsubject String@@ -81,34 +83,15 @@ model Promotion {audience Json @default("{}")// Earlier promotions whose recipients are skipped.excludePromotionIds String[] @default([])- // "draft" | "sending" | "sent"+ // "draft" | "sending" | "paused" | "sent"status String @default("draft")startedAt DateTime?finishedAt DateTime?actorUserId String?createdAt DateTime @default(now())updatedAt DateTime @updatedAt- recipients PromotionRecipient[]+ sends EmailSend[]@@index([status, createdAt])}-// One row per account a promotion resolved to. sentAt marks the send; error-// holds why one did not go out. The pair is unique, so a retried send can-// never mail an account twice.-model PromotionRecipient {- id String @id @default(cuid())- promotionId String- promotion Promotion @relation(fields: [promotionId], references: [id], onDelete: Cascade)- userId String- user User @relation(fields: [userId], references: [id], onDelete: Cascade)- sentAt DateTime?- // Sends tried so far; the last error stays on the row. A row is given up- // after MAX_SEND_ATTEMPTS (src/lib/marketing/promotionInput.ts).- attempts Int @default(0)- error String?- createdAt DateTime @default(now())-- @@unique([promotionId, userId])- @@index([promotionId, sentAt])-}diff --git a/site/prisma/migrations/20260910024516/migration.sql b/site/prisma/migrations/20260910024516/migration.sqlnew file mode 100644index 00000000..265f0985--- /dev/null+++ b/site/prisma/migrations/20260910024516/migration.sql@@ -0,0 +1,54 @@+/*+ Warnings:++ - You are about to drop the column `inFlight` on the `EmailDailyQuota` table. All the data in the column will be lost.+ - You are about to drop the `PromotionRecipient` table. If the table is not empty, all the data it contains will be lost.++*/+-- DropForeignKey+ALTER TABLE "PromotionRecipient" DROP CONSTRAINT "PromotionRecipient_promotionId_fkey";++-- DropForeignKey+ALTER TABLE "PromotionRecipient" DROP CONSTRAINT "PromotionRecipient_userId_fkey";++-- AlterTable+ALTER TABLE "EmailDailyQuota" DROP COLUMN "inFlight";++-- DropTable+DROP TABLE "PromotionRecipient";++-- CreateTable+CREATE TABLE "EmailSend" (+ "id" TEXT NOT NULL,+ "kind" TEXT NOT NULL,+ "priority" INTEGER NOT NULL,+ "rank" INTEGER NOT NULL DEFAULT 0,+ "userId" TEXT,+ "promotionId" TEXT,+ "payload" JSONB NOT NULL,+ "idempotencyKey" TEXT NOT NULL,+ "state" TEXT NOT NULL DEFAULT 'queued',+ "attempts" INTEGER NOT NULL DEFAULT 0,+ "error" TEXT,+ "notBefore" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "sentAt" TIMESTAMP(3),+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "updatedAt" TIMESTAMP(3) NOT NULL,++ CONSTRAINT "EmailSend_pkey" PRIMARY KEY ("id")+);++-- CreateIndex+CREATE UNIQUE INDEX "EmailSend_idempotencyKey_key" ON "EmailSend"("idempotencyKey");++-- CreateIndex+CREATE INDEX "EmailSend_state_priority_rank_createdAt_idx" ON "EmailSend"("state", "priority", "rank", "createdAt");++-- CreateIndex+CREATE INDEX "EmailSend_promotionId_state_idx" ON "EmailSend"("promotionId", "state");++-- AddForeignKey+ALTER TABLE "EmailSend" ADD CONSTRAINT "EmailSend_userId_fkey" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE CASCADE;++-- AddForeignKey+ALTER TABLE "EmailSend" ADD CONSTRAINT "EmailSend_promotionId_fkey" FOREIGN KEY ("promotionId") REFERENCES "Promotion"("id") ON DELETE CASCADE ON UPDATE CASCADE;diff --git a/site/src/app/api/credits/offers/route.ts b/site/src/app/api/credits/offers/route.tsindex b19ac5fd..6cbea9f7 100644--- a/site/src/app/api/credits/offers/route.ts+++ b/site/src/app/api/credits/offers/route.ts@@ -45,16 +45,20 @@ export const POST = withSuperUser(async (request) => {});if (!targetUser) return notFoundResponse();- const offer = await createCreditOffer({+ const { delivery, offer } = await createCreditOffer({amountMicros: creditStringToMicros(String(parsed.data.amountDollars)),description: parsed.data.description,expiresAfterDays: parsed.data.expiresAfterDays,offeredByUserId: request.donkey.userId,user: targetUser,});+ if (delivery.state === "failed") {+ return NextResponse.json({ error: "not_sendable", message: delivery.error }, { status: 409 });+ }return NextResponse.json({amountDollars: parsed.data.amountDollars,+ delivery: delivery.state,offer: {closesAt: offer.closesAt?.toISOString() ?? null,emailSentAt: offer.emailSentAt?.toISOString() ?? null,diff --git a/site/src/app/api/marketing/outreach/route.ts b/site/src/app/api/marketing/outreach/route.tsindex cbca4670..4717f3d5 100644--- a/site/src/app/api/marketing/outreach/route.ts+++ b/site/src/app/api/marketing/outreach/route.ts@@ -9,11 +9,9 @@ import {OUTREACH_REASONS,OUTREACH_STATUSES,} from "@/lib/marketing/campaigns";+import { deliverEmail } from "@/lib/email/outbox";import { firstNameOf } from "@/lib/marketing/placeholders";-import {- OutreachNotSendableError,- sendOutreachEmail,-} from "@/lib/marketing/send-outreach";+import { outreachIdempotencyKey } from "@/lib/marketing/send-outreach";import { prisma } from "@/lib/prisma";const listQuerySchema = z.object({@@ -206,15 +204,18 @@ export const POST = withSuperUser(async (request) => {const actorUserId = request.donkey.userId;if (parsed.data.action === "send") {- try {- await sendOutreachEmail({- attempt: outreach.sentCount + 1,+ const attempt = outreach.sentCount + 1;+ const delivery = await deliverEmail({+ idempotencyKey: outreachIdempotencyKey(outreach.id, attempt),+ kind: "outreach",+ payload: {+ actorUserId,+ attempt,body: parsed.data.body,outreachId: outreach.id,subject: parsed.data.subject,trackReplies: parsed.data.trackReplies,unsubscribeLink: parsed.data.unsubscribeLink,- user: outreach.user,vars: {balance: creditMicrosToString(outreach.balanceMicros),email: outreach.user.email,@@ -223,28 +224,14 @@ export const POST = withSuperUser(async (request) => {spent: creditMicrosToString(outreach.spentMicros),storage: formatBytes(Number(outreach.storageBytes)),},- });- } catch (error) {- if (error instanceof OutreachNotSendableError) {- return NextResponse.json(- { error: "not_sendable", message: error.message },- { status: 409 },- );- }- throw error;- }- const row = await prisma.userOutreach.update({- data: {- actorUserId,- firstSentAt: outreach.firstSentAt ?? now,- lastSentAt: now,- sentCount: { increment: 1 },- status: "sent",},- select: rowSelect,- where: { id: outreach.id },+ userId: outreach.user.id,});- return NextResponse.json({ row: serialize(row) });+ if (delivery.state === "failed") {+ return NextResponse.json({ error: "not_sendable", message: delivery.error }, { status: 409 });+ }+ const row = await prisma.userOutreach.findUniqueOrThrow({ select: rowSelect, where: { id: outreach.id } });+ return NextResponse.json({ delivery: delivery.state, row: serialize(row) });}const data =diff --git a/site/src/app/api/su/promotions/[id]/send/route.ts b/site/src/app/api/su/promotions/[id]/send/route.tsindex 3ee1a3fa..463366a9 100644--- a/site/src/app/api/su/promotions/[id]/send/route.ts+++ b/site/src/app/api/su/promotions/[id]/send/route.ts@@ -3,7 +3,8 @@ import { z } from "zod";import { invalidResponse } from "@/lib/config/experimentList";import { notFoundResponse, withSuperUser } from "@/lib/donkey-api-auth";-import { enqueueJob } from "@/lib/jobs/queue";+import { queueEmails } from "@/lib/email/outbox";+import { byMostRecentlyActive, lastActiveByUser } from "@/lib/marketing/lastActive";import {listPromotions,promotionAudienceOf,@@ -19,16 +20,16 @@ export const maxDuration = 300;type Params = { params: Promise<{ id: string }> };const idSchema = z.string().trim().min(1);-// Starts a send: the segment is resolved into recipient rows once, here, and-// the promotion-send job mails them. The rows are the send's memory — a later-// promotion that excludes this one reads them, and a re-run of the job skips-// the ones already marked sent.+// Starts a send: the segment is resolved into outbox rows once, here, and the+// outbox drainer mails them under the day's quota. The rows are the send's+// memory — a later promotion that excludes this one reads them, and a resumed+// send adds only the accounts not already queued.export const POST = withSuperUser(async (request, { params }: Params) => {const id = idSchema.safeParse((await params).id);if (!id.success) return notFoundResponse();const promotion = await prisma.promotion.findUnique({ where: { id: id.data } });if (!promotion) return notFoundResponse();- if (promotion.status !== "draft") {+ if (promotion.status !== "draft" && promotion.status !== "paused") {return invalidResponse([{ path: ["status"], message: "This promotion has already been sent." }]);}@@ -40,20 +41,26 @@ export const POST = withSuperUser(async (request, { params }: Params) => {return invalidResponse([{ path: ["audience"], message: "Nobody is in this segment." }]);}- await prisma.$transaction([- prisma.promotionRecipient.createMany({- data: segment.users.map((user) => ({ promotionId: promotion.id, userId: user.id })),- skipDuplicates: true,- }),- prisma.promotion.update({- data: { startedAt: new Date(), status: "sending" },- where: { id: promotion.id },- }),- ]);- const { jobId } = await enqueueJob("promotion-send", { promotionId: promotion.id }, request.donkey.userId);+ // The people most recently using the product go first: a campaign that+ // outlasts the day's send quota reaches them before it reaches the+ // dormant.+ const ranked = byMostRecentlyActive(segment.users, await lastActiveByUser(segment.users.map((u) => u.id)));+ await prisma.promotion.update({+ data: { startedAt: promotion.startedAt ?? new Date(), status: "sending" },+ where: { id: promotion.id },+ });+ await queueEmails(+ ranked.map((user, rank) => ({+ idempotencyKey: `promotion:${promotion.id}:${user.id}`,+ kind: "promotion",+ payload: { promotionId: promotion.id },+ promotionId: promotion.id,+ rank,+ userId: user.id,+ })),+ );return NextResponse.json({- jobId,promotions: await listPromotions(),recipients: segment.users.length,});diff --git a/site/src/app/api/su/promotions/[id]/test/route.ts b/site/src/app/api/su/promotions/[id]/test/route.tsindex 1bb6a8b6..300754ac 100644--- a/site/src/app/api/su/promotions/[id]/test/route.ts+++ b/site/src/app/api/su/promotions/[id]/test/route.ts@@ -3,38 +3,29 @@ import { z } from "zod";import { invalidResponse } from "@/lib/config/experimentList";import { notFoundResponse, withSuperUser } from "@/lib/donkey-api-auth";-import { UnknownPlaceholderError } from "@/lib/marketing/placeholders";-import { promotionCopyOf, sendPromotionEmail } from "@/lib/marketing/promotions";+import { deliverEmail } from "@/lib/email/outbox";import { prisma } from "@/lib/prisma";type Params = { params: Promise<{ id: string }> };const idSchema = z.string().trim().min(1);// Mails the saved copy to the operator, placeholders filled with their own-// account, from the sender the promotion names. Every click sends again.+// account. A copy error comes back on the body field.export const POST = withSuperUser(async (request, { params }: Params) => {const id = idSchema.safeParse((await params).id);if (!id.success) return notFoundResponse();const [promotion, operator] = await Promise.all([- prisma.promotion.findUnique({ where: { id: id.data } }),- prisma.user.findUnique({- select: { email: true, id: true, name: true },- where: { id: request.donkey.userId },- }),+ prisma.promotion.findUnique({ select: { id: true }, where: { id: id.data } }),+ prisma.user.findUnique({ select: { email: true, id: true }, where: { id: request.donkey.userId } }),]);if (!promotion || !operator) return notFoundResponse();- try {- await sendPromotionEmail({- copy: promotionCopyOf(promotion),- idempotencyKey: `promotion-test:${promotion.id}:${Date.now()}`,- user: operator,- });- } catch (error) {- if (error instanceof UnknownPlaceholderError) {- return invalidResponse([{ path: ["body"], message: error.message }]);- }- throw error;- }- return NextResponse.json({ sentTo: operator.email });+ const delivery = await deliverEmail({+ idempotencyKey: `promotion-test:${promotion.id}:${Date.now()}`,+ kind: "promotion-test",+ payload: { promotionId: promotion.id },+ userId: operator.id,+ });+ if (delivery.state === "failed") return invalidResponse([{ path: ["body"], message: delivery.error ?? "The send failed." }]);+ return NextResponse.json({ delivery: delivery.state, sentTo: operator.email });});diff --git a/site/src/lib/credits/offers.ts b/site/src/lib/credits/offers.tsindex d9a8d0b5..35b36ceb 100644--- a/site/src/lib/credits/offers.ts+++ b/site/src/lib/credits/offers.ts@@ -6,7 +6,8 @@ import { getGlobalSetting } from "@/lib/config/effective";import { DONKEYCUT_CANONICAL } from "@/cut/lib/hosts";import { grantCredits } from "@/lib/credits/inference";import { creditGrantExpiry } from "@/lib/credits/top-up";-import { sendCreditsOfferedEmail } from "@/lib/email/send-credits-offered";+import { deliverEmail } from "@/lib/email/outbox";+import { creditsOfferedIdempotencyKey } from "@/lib/email/send-credits-offered";import { prisma } from "@/lib/prisma";// Claim links authenticate with an HMAC over the offer id, keyed by a@@ -85,17 +86,14 @@ export async function createCreditOffer(input: {userId: input.user.id,},}));- await sendCreditsOfferedEmail(input.user, {- amountMicros: offer.amountMicros,- claimUrl: creditOfferClaimUrl(offer.id),- closesAt,- expiresAfterDays: offer.expiresAfterDays,- id: offer.id,- });- return prisma.creditOffer.update({- data: { closesAt, emailSentAt: new Date() },- where: { id: offer.id },+ await prisma.creditOffer.update({ data: { closesAt }, where: { id: offer.id } });+ const delivery = await deliverEmail({+ idempotencyKey: creditsOfferedIdempotencyKey(offer.id),+ kind: "credit-offer",+ payload: { offerId: offer.id },+ userId: input.user.id,});+ return { delivery, offer: await prisma.creditOffer.findUniqueOrThrow({ where: { id: offer.id } }) };}export class CreditOfferNotYoursError extends Error {}@@ -139,7 +137,7 @@ export async function landCreditOffer(// promotion's offer; those land from the act they reward.export async function claimCreditOffer(offerId: string, userId: string) {const offer = await prisma.creditOffer.findUnique({ where: { id: offerId } });- if (!offer || offer.kind !== MANUAL_OFFER_KIND) return null;+ if (!offer || (offer.kind !== MANUAL_OFFER_KIND && offer.kind !== "promotion_email")) return null;if (offer.userId !== userId) throw new CreditOfferNotYoursError();if (!offer.claimedAt && !creditOfferOpen(offer, new Date())) throw new CreditOfferClosedError();return landCreditOffer(offer, {@@ -148,3 +146,31 @@ export async function claimCreditOffer(offerId: string, userId: string) {sourceId: `offer:${offer.id}`,});}++export async function createPromotionCreditOffer(input: {+ promotionId: string;+ userId: string;+ amountDollars: number;+ claimWindowDays: number;+ expiresAfterDays: number;+ offeredByUserId: string | null;+ now?: Date;+}) {+ const now = input.now ?? new Date();+ const offerId = `${input.promotionId}:${input.userId}`;+ const offer = await prisma.creditOffer.upsert({+ where: { id: offerId },+ create: {+ id: offerId,+ userId: input.userId,+ kind: "promotion_email",+ amountMicros: BigInt(input.amountDollars) * BigInt(1_000_000),+ expiresAfterDays: input.expiresAfterDays,+ closesAt: new Date(now.getTime() + input.claimWindowDays * 86_400_000),+ description: "Promotional AI credits",+ offeredByUserId: input.offeredByUserId,+ },+ update: {},+ });+ return { offer, claimUrl: creditOfferClaimUrl(offer.id) };+}diff --git a/site/src/lib/email/credit-expiry-notices.ts b/site/src/lib/email/credit-expiry-notices.tsindex 11c7a344..711ed6c6 100644--- a/site/src/lib/email/credit-expiry-notices.ts+++ b/site/src/lib/email/credit-expiry-notices.ts@@ -1,77 +1,36 @@-import type { Prisma } from "@/generated/prisma/client";-import { creditMicrosToString, zeroCreditMicros } from "@/lib/credits/amounts";-import { bulkFrom, isResendConfigured, ResendNotConfiguredError } from "@/lib/email/resend";-import { sendCreditsExpiringEmail } from "@/lib/email/send-credits-expiring";+import { creditMicrosToString } from "@/lib/credits/amounts";+import { metadataObject, pendingCreditExpiryGrants } from "@/lib/email/credit-expiry-due";+import { queueEmails } from "@/lib/email/outbox";+import { creditsExpiringIdempotencyKey } from "@/lib/email/send-credits-expiring";import { prisma } from "@/lib/prisma";-// The grants an account is warned about: credit it was given. Purchased-// credit never expires, and the Pro allowance turns over every month.-export const NOTICED_GRANT_SOURCES = ["signup", "manual_dollar", "subscribe_bonus"] as const;--const DAY_MS = 24 * 60 * 60 * 1000;--function metadataObject(metadata: Prisma.JsonValue | null): Prisma.JsonObject {- return metadata !== null && typeof metadata === "object" && !Array.isArray(metadata)- ? metadata- : {};-}--/** Whether the grant's account has already been told this grant expires. */-export function expiryNoticed(metadata: Prisma.JsonValue | null): boolean {- return typeof metadataObject(metadata).expiryNoticeSentAt === "string";+// Queues one expiry notice for every account holding a given grant that+// expires inside the window and has not been told. The outbox row is keyed+// on the grant, so a run that overlaps the last one adds nothing; the send+// is recorded on the grant's metadata once it goes out. Credit on the+// account is account mail, so the marketing opt-out does not apply; the+// footer still offers it.+export async function queueDueCreditExpiryNotices(input: { now?: Date; withinDays: number }) {+ const now = input.now ?? new Date();+ const { due, pending } = await pendingCreditExpiryGrants(now, input.withinDays);+ const queued = await queueEmails(+ pending+ .filter((grant) => grant.expiresAt)+ .map((grant) => ({+ idempotencyKey: creditsExpiringIdempotencyKey(grant.id),+ kind: "credit-expiry",+ payload: { credits: creditMicrosToString(grant.remainingAmountMicros), grantId: grant.id },+ userId: grant.user.id,+ })),+ );+ return { due, pending: pending.length, queued };}-// Emails every account holding a given grant that expires inside the window,-// once per grant: the send is recorded on the grant's metadata, so the next-// run reaches only what this one missed. Credit on the account is account-// mail, so the marketing opt-out does not apply; the footer still offers it.-export async function sendDueCreditExpiryNotices(input: { now?: Date; withinDays: number }) {- if (!isResendConfigured()) throw new ResendNotConfiguredError();- if (!bulkFrom()) throw new Error("No bulk sender configured.");- const now = input.now ?? new Date();- const grants = await prisma.userCreditGrant.findMany({- orderBy: { expiresAt: "asc" },- select: {- id: true,- expiresAt: true,- metadata: true,- remainingAmountMicros: true,- user: { select: { email: true, id: true, name: true } },- },- where: {- expiresAt: { gt: now, lte: new Date(now.getTime() + input.withinDays * DAY_MS) },- remainingAmountMicros: { gt: zeroCreditMicros },- source: { in: [...NOTICED_GRANT_SOURCES] },- status: "active",- },+export async function recordExpiryNoticeSent(grantId: string): Promise<void> {+ const grant = await prisma.userCreditGrant.findUnique({ select: { metadata: true }, where: { id: grantId } });+ if (!grant) return;+ await prisma.userCreditGrant.update({+ data: { metadata: { ...metadataObject(grant.metadata), expiryNoticeSentAt: new Date().toISOString() } },+ where: { id: grantId },});- const pending = grants.filter((g) => !expiryNoticed(g.metadata));-- let sent = 0;- let failed = 0;- for (const grant of pending) {- if (!grant.expiresAt) continue;- try {- await sendCreditsExpiringEmail(- grant.user,- creditMicrosToString(grant.remainingAmountMicros),- grant.expiresAt,- `credits-expiring:${grant.id}`,- );- await prisma.userCreditGrant.update({- data: {- metadata: {- ...metadataObject(grant.metadata),- expiryNoticeSentAt: new Date().toISOString(),- },- },- where: { id: grant.id },- });- sent++;- } catch (error) {- failed++;- console.error("[credit-expiry-notice] failed to send", { error, userId: grant.user.id });- }- }- return { due: grants.length, failed, pending: pending.length, sent };}diff --git a/site/src/lib/email/errors.ts b/site/src/lib/email/errors.tsnew file mode 100644index 00000000..5e976f76--- /dev/null+++ b/site/src/lib/email/errors.ts@@ -0,0 +1,8 @@+// Thrown by a builder when the email can never go out as queued: the row is+// given up without a retry and the message lands on it.+export class PermanentSendError extends Error {+ public constructor(message: string) {+ super(message);+ this.name = "PermanentSendError";+ }+}diff --git a/site/src/lib/email/kindIds.ts b/site/src/lib/email/kindIds.tsnew file mode 100644index 00000000..1a726955--- /dev/null+++ b/site/src/lib/email/kindIds.ts@@ -0,0 +1,27 @@+// The kinds of email the site sends, and where each stands in the queue. The+// server side of a kind (what it builds, whose quota it spends) is in+// src/lib/email/kinds.ts; this file is client-safe so the settings registry+// can offer the priorities to su.+export const EMAIL_KIND_IDS = [+ "reply-forward",+ "credit-offer",+ "outreach",+ "promotion-test",+ "welcome",+ "credit-expiry",+ "promotion",+] as const;++export type EmailKindId = (typeof EMAIL_KIND_IDS)[number];++// Higher goes first. A reply from a user beats everything; a campaign yields+// to every email a single account is waiting on.+export const DEFAULT_EMAIL_PRIORITIES: Record<EmailKindId, number> = {+ "reply-forward": 100,+ "credit-offer": 90,+ outreach: 80,+ "promotion-test": 70,+ welcome: 60,+ "credit-expiry": 50,+ promotion: 10,+};diff --git a/site/src/lib/email/kinds.ts b/site/src/lib/email/kinds.tsnew file mode 100644index 00000000..da879137--- /dev/null+++ b/site/src/lib/email/kinds.ts@@ -0,0 +1,174 @@+import { z } from "zod";++import { creditOfferClaimUrl, createPromotionCreditOffer } from "@/lib/credits/offers";+import { recordExpiryNoticeSent } from "@/lib/email/credit-expiry-notices";+import type { EmailSendKind } from "@/lib/email/daily-send-limit";+import { PermanentSendError } from "@/lib/email/errors";+import type { EmailKindId } from "@/lib/email/kindIds";+import type { EmailMessage, EmailUser } from "@/lib/email/resend";+import { buildCreditsExpiringEmail } from "@/lib/email/send-credits-expiring";+import { buildCreditsOfferedEmail } from "@/lib/email/send-credits-offered";+import { buildWelcomeEmail, recordWelcomeSent } from "@/lib/email/send-welcome";+import { isMarketingUnsubscribed } from "@/lib/email/unsubscribe";+import { buildReplyForwardEmail, replyForwardPayloadSchema } from "@/lib/marketing/forward-reply";+import { UnknownPlaceholderError } from "@/lib/marketing/placeholders";+import { SAMPLE_CLAIM_URL, promotionOfferOf } from "@/lib/marketing/promotionOfferInput";+import { buildPromotionEmail, promotionCopyOf } from "@/lib/marketing/promotions";+import { buildOutreachEmail, outreachPayloadSchema, recordOutreachSent } from "@/lib/marketing/send-outreach";+import { prisma } from "@/lib/prisma";++// What each kind of email is on the server: whose quota it spends, what its+// payload holds, how the message is built, and what to write down once it+// went. A new kind is an entry here plus its id and priority in kindIds.ts.++export type OutboxRow = {+ id: string;+ userId: string | null;+ promotionId: string | null;+ idempotencyKey: string;+};++export type EmailKind<P = unknown> = {+ quota: EmailSendKind;+ payload: z.ZodType<P>;+ // The message for the provider, or null when nothing should go out any+ // more (the account is gone or opted out): the row is dropped as skipped.+ build: (payload: P, row: OutboxRow) => Promise<EmailMessage | null>;+ // Bookkeeping once the provider accepted the message.+ afterSend?: (payload: P, row: OutboxRow) => Promise<void>;+ // Bookkeeping when the row is given up for good.+ onAbandon?: (payload: P, row: OutboxRow, error: Error) => Promise<void>;+};++function define<P>(kind: EmailKind<P>): EmailKind<P> {+ return kind;+}++const USER_SELECT = { email: true, id: true, name: true } as const;++async function userOf(row: OutboxRow): Promise<EmailUser | null> {+ if (!row.userId) throw new PermanentSendError("This email needs an account.");+ return prisma.user.findUnique({ select: USER_SELECT, where: { id: row.userId } });+}++const welcome = define({+ quota: "transactional",+ payload: z.object({ credits: z.string().nullable() }).strict(),+ build: async ({ credits }, row) => {+ const user = await userOf(row);+ return user ? buildWelcomeEmail(user, credits) : null;+ },+ afterSend: async (_payload, row) => recordWelcomeSent(row.userId!),+});++const creditExpiry = define({+ quota: "transactional",+ payload: z.object({ credits: z.string(), grantId: z.string().min(1) }).strict(),+ build: async ({ credits, grantId }, row) => {+ const [user, grant] = await Promise.all([+ userOf(row),+ prisma.userCreditGrant.findUnique({ select: { expiresAt: true, status: true }, where: { id: grantId } }),+ ]);+ if (!user || !grant?.expiresAt || grant.status !== "active") return null;+ return buildCreditsExpiringEmail(user, credits, grant.expiresAt);+ },+ afterSend: async ({ grantId }) => recordExpiryNoticeSent(grantId),+});++const creditOffer = define({+ quota: "manual",+ payload: z.object({ offerId: z.string().min(1) }).strict(),+ build: async ({ offerId }, row) => {+ const [user, offer] = await Promise.all([userOf(row), prisma.creditOffer.findUnique({ where: { id: offerId } })]);+ if (!user || !offer || offer.claimedAt || !offer.closesAt) return null;+ return buildCreditsOfferedEmail(user, {+ amountMicros: offer.amountMicros,+ claimUrl: creditOfferClaimUrl(offer.id),+ closesAt: offer.closesAt,+ expiresAfterDays: offer.expiresAfterDays,+ });+ },+ afterSend: async ({ offerId }) => {+ await prisma.creditOffer.update({ data: { emailSentAt: new Date() }, where: { id: offerId } });+ },+});++const outreach = define({+ quota: "manual",+ payload: outreachPayloadSchema,+ build: async (payload, row) => {+ const user = await userOf(row);+ return user ? buildOutreachEmail(payload, user) : null;+ },+ afterSend: async (payload) => recordOutreachSent(payload),+});++const replyForward = define({+ quota: "manual",+ payload: replyForwardPayloadSchema,+ build: (payload) => buildReplyForwardEmail(payload),+});++const promotionPayloadSchema = z.object({ promotionId: z.string().min(1) }).strict();++async function promotionRowOf(promotionId: string) {+ const promotion = await prisma.promotion.findUnique({ where: { id: promotionId } });+ if (!promotion) throw new PermanentSendError("The promotion no longer exists.");+ return promotion;+}++// A copy error is permanent for the whole send: the draft comes back so the+// operator can fix it, and the unsent rows go with it.+async function abandonPromotion(promotionId: string, error: Error): Promise<void> {+ if (!(error instanceof PermanentSendError || error instanceof UnknownPlaceholderError)) return;+ await prisma.emailSend.deleteMany({ where: { promotionId, state: "queued" } });+ await prisma.promotion.update({ data: { status: "draft" }, where: { id: promotionId } });+}++const promotion = define({+ quota: "bulk",+ payload: promotionPayloadSchema,+ build: async ({ promotionId }, row) => {+ const [user, promotion] = await Promise.all([userOf(row), promotionRowOf(promotionId)]);+ // The opt-out is read again right before the send, so someone who+ // unsubscribed after the segment was resolved is skipped.+ if (!user || (await isMarketingUnsubscribed(user.id))) return null;+ const offer = promotionOfferOf(promotion.creditOffer);+ const claimUrl = offer+ ? (+ await createPromotionCreditOffer({+ promotionId,+ userId: user.id,+ amountDollars: offer.dollars,+ claimWindowDays: offer.claimWindowDays,+ expiresAfterDays: offer.expiresAfterDays,+ offeredByUserId: promotion.actorUserId,+ })+ ).claimUrl+ : undefined;+ return buildPromotionEmail(promotionCopyOf(promotion), user, claimUrl);+ },+ onAbandon: async ({ promotionId }, _row, error) => abandonPromotion(promotionId, error),+});++// The saved copy mailed to the operator, placeholders filled with their own+// account and a sample claim link where the copy has one.+const promotionTest = define({+ quota: "manual",+ payload: promotionPayloadSchema,+ build: async ({ promotionId }, row) => {+ const [user, promotion] = await Promise.all([userOf(row), promotionRowOf(promotionId)]);+ if (!user) return null;+ return buildPromotionEmail(promotionCopyOf(promotion), user, promotion.creditOffer ? SAMPLE_CLAIM_URL : undefined);+ },+});++export const EMAIL_KINDS: Record<EmailKindId, EmailKind> = {+ "reply-forward": replyForward as EmailKind,+ "credit-offer": creditOffer as EmailKind,+ outreach: outreach as EmailKind,+ "promotion-test": promotionTest as EmailKind,+ welcome: welcome as EmailKind,+ "credit-expiry": creditExpiry as EmailKind,+ promotion: promotion as EmailKind,+};diff --git a/site/src/lib/email/outbox.ts b/site/src/lib/email/outbox.tsnew file mode 100644index 00000000..c642f4a4--- /dev/null+++ b/site/src/lib/email/outbox.ts@@ -0,0 +1,348 @@+import type { Prisma } from "@/generated/prisma/client";+import { getGlobalSetting } from "@/lib/config/effective";+import {+ DailyEmailSendLimitError,+ dailyEmailQuotaStatus,+ type DailyEmailQuotaStatus,+ type EmailSendKind,+ withDailyEmailQuota,+} from "@/lib/email/daily-send-limit";+import { PermanentSendError } from "@/lib/email/errors";+import { EMAIL_KIND_IDS, type EmailKindId } from "@/lib/email/kindIds";+import { EMAIL_KINDS, type OutboxRow } from "@/lib/email/kinds";+import { getResend, isResendConfigured, ResendNotConfiguredError } from "@/lib/email/resend";+import { UnknownPlaceholderError } from "@/lib/marketing/placeholders";+import { prisma } from "@/lib/prisma";++// The outbox. Every email is queued as a row and sent by the drainer in+// priority order, each row under its kind's share of the day's quota. An+// interactive send (an operator's note, a signup's welcome) is queued and+// tried at once; when the quota refuses it, it waits its turn. The row is+// the record: what went, what failed and why, and what is still owed.++const PAGE = 50;+// Provider rate limit: two requests a second.+const PACE_MS = 550;+const MAX_ATTEMPTS = 5;+// A row claimed this long ago and never settled is a killed drainer's.+const STALE_SENDING_MS = 15 * 60_000;++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++export type QueueEmailInput = {+ kind: EmailKindId;+ payload: Prisma.InputJsonValue;+ idempotencyKey: string;+ userId?: string;+ promotionId?: string;+ rank?: number;+};++export type DeliverResult = { id: string; state: "sent" | "queued" | "failed" | "skipped"; error: string | null };++type Outcome =+ | { outcome: "sent" | "retried" | "failed" | "skipped" | "busy" }+ | { outcome: "refused"; retryAfterSeconds: number };++function claimable(now: Date): Prisma.EmailSendWhereInput {+ return {+ OR: [{ state: "queued" }, { state: "sending", updatedAt: { lt: new Date(now.getTime() - STALE_SENDING_MS) } }],+ };+}++function isPermanent(error: unknown): boolean {+ return (+ error instanceof PermanentSendError ||+ error instanceof UnknownPlaceholderError ||+ error instanceof ResendNotConfiguredError+ );+}++function messageOf(error: unknown): string {+ return error instanceof Error ? error.message : String(error);+}++async function rowsOf(inputs: QueueEmailInput[]) {+ const priorities = await getGlobalSetting("emailPriorities");+ return inputs.map((input) => ({+ idempotencyKey: input.idempotencyKey,+ kind: input.kind,+ payload: input.payload,+ priority: priorities[input.kind],+ promotionId: input.promotionId,+ rank: input.rank ?? 0,+ userId: input.userId,+ }));+}++/** Queues many emails at once; rows already queued under the same key are+ * left as they are. Returns how many were new. */+export async function queueEmails(inputs: QueueEmailInput[]): Promise<number> {+ if (inputs.length === 0) return 0;+ const { count } = await prisma.emailSend.createMany({ data: await rowsOf(inputs), skipDuplicates: true });+ if (count > 0) await scheduleDrain(0);+ return count;+}++/** Queues one email and tries to send it now. When the quota refuses it, the+ * row waits for the drainer; a permanent failure is reported on the result. */+export async function deliverEmail(input: QueueEmailInput): Promise<DeliverResult> {+ const [data] = await rowsOf([input]);+ await prisma.emailSend.createMany({ data: [data], skipDuplicates: true });+ const row = await prisma.emailSend.findUniqueOrThrow({ where: { idempotencyKey: input.idempotencyKey } });+ if (row.state === "queued") {+ const outcome = await sendRow(row, new Date());+ if (outcome.outcome === "refused") await scheduleDrain(outcome.retryAfterSeconds);+ }+ const settled = await prisma.emailSend.findUniqueOrThrow({ select: { error: true, id: true, state: true }, where: { id: row.id } });+ const state = settled.state === "sending" ? "queued" : (settled.state as DeliverResult["state"]);+ return { error: settled.error, id: settled.id, state };+}++/** Makes sure a drainer is coming, no sooner than the delay. One pending+ * drain job serves every caller. */+export async function scheduleDrain(delaySeconds: number): Promise<void> {+ const pending = await prisma.asyncJob.findFirst({+ select: { id: true },+ where: { kind: "email-drain", state: { in: ["queued", "running"] } },+ });+ if (pending) return;+ // Imported here because the job registry imports this module.+ const { enqueueJob } = await import("@/lib/jobs/queue");+ await enqueueJob("email-drain", {}, "system", { delaySeconds });+}++type DrainableRow = Awaited<ReturnType<typeof prisma.emailSend.findMany>>[number];++async function sendRow(row: DrainableRow, now: Date): Promise<Outcome> {+ const claimed = await prisma.emailSend.updateMany({+ data: { state: "sending" },+ where: { id: row.id, ...claimable(now) },+ });+ if (claimed.count === 0) return { outcome: "busy" };++ const outboxRow: OutboxRow = { id: row.id, idempotencyKey: row.idempotencyKey, promotionId: row.promotionId, userId: row.userId };+ const kind = EMAIL_KINDS[row.kind as EmailKindId];+ const attempts = row.attempts + 1;+ const settle = (data: Prisma.EmailSendUpdateInput) => prisma.emailSend.update({ data, where: { id: row.id } });++ const giveUp = async (error: unknown, payload: unknown) => {+ await settle({ attempts, error: messageOf(error), state: "failed" });+ if (kind && payload !== undefined) await kind.onAbandon?.(payload, outboxRow, error instanceof Error ? error : new Error(messageOf(error)));+ return { outcome: "failed" } as const;+ };++ if (!kind) return giveUp(new PermanentSendError(`Unknown email kind "${row.kind}".`), undefined);+ const parsed = kind.payload.safeParse(row.payload);+ if (!parsed.success) return giveUp(new PermanentSendError("The stored payload is invalid."), undefined);+ const payload = parsed.data;++ try {+ if (!isResendConfigured()) throw new ResendNotConfiguredError();+ const message = await kind.build(payload, outboxRow);+ if (!message) {+ await settle({ attempts, error: null, state: "skipped" });+ return { outcome: "skipped" };+ }+ const result = await withDailyEmailQuota(kind.quota, () =>+ getResend().emails.send(message, { idempotencyKey: row.idempotencyKey }),+ );+ if (result.error) throw new Error(`Resend send failed: ${result.error.name}: ${result.error.message}`);+ await settle({ attempts, error: null, sentAt: new Date(), state: "sent" });+ await kind.afterSend?.(payload, outboxRow);+ return { outcome: "sent" };+ } catch (error) {+ if (error instanceof DailyEmailSendLimitError) {+ await settle({ state: "queued" });+ return { outcome: "refused", retryAfterSeconds: error.retryAfterSeconds };+ }+ if (isPermanent(error) || attempts >= MAX_ATTEMPTS) return giveUp(error, payload);+ // A provider blip: back off, further each time, so a bad hour does not+ // burn the row's attempts.+ await settle({+ attempts,+ error: messageOf(error),+ notBefore: new Date(now.getTime() + attempts * 5 * 60_000),+ state: "queued",+ });+ return { outcome: "retried" };+ }+}++export type DrainResult = {+ sent: number;+ failed: number;+ skipped: number;+ retried: number;+ // Seconds until a refused quota is expected to open; null when nothing was refused.+ retryAfterSeconds: number | null;+ // Whether the budget ran out with rows still owed.+ more: boolean;+};++/** Sends what the outbox holds, highest priority first, until the budget or+ * the quota runs out. A refused quota class is left alone for the rest of+ * the run; the classes above it keep going. */+export async function drainOutbox(budgetMs: number): Promise<DrainResult> {+ const startedAt = Date.now();+ const result: DrainResult = { sent: 0, failed: 0, skipped: 0, retried: 0, retryAfterSeconds: null, more: false };+ const refused = new Map<EmailSendKind, number>();+ const touchedPromotions = new Set<string>();++ const refusedKinds = () =>+ (Object.keys(EMAIL_KINDS) as EmailKindId[]).filter((id) => refused.has(EMAIL_KINDS[id].quota));++ drain: for (;;) {+ const now = new Date();+ const rows = await prisma.emailSend.findMany({+ orderBy: [{ priority: "desc" }, { rank: "asc" }, { createdAt: "asc" }],+ take: PAGE,+ where: {+ AND: [+ claimable(now),+ { kind: { notIn: refusedKinds() }, notBefore: { lte: now } },+ // A paused promotion's rows wait where they are.+ { OR: [{ promotionId: null }, { promotion: { status: "sending" } }] },+ ],+ },+ });+ if (rows.length === 0) break;+ for (const row of rows) {+ if (Date.now() - startedAt > budgetMs) {+ result.more = true;+ break drain;+ }+ const quota = EMAIL_KINDS[row.kind as EmailKindId]?.quota;+ if (quota && refused.has(quota)) continue;+ const outcome = await sendRow(row, new Date());+ if (row.promotionId) touchedPromotions.add(row.promotionId);+ if (outcome.outcome === "refused") {+ if (quota) refused.set(quota, outcome.retryAfterSeconds);+ continue;+ }+ if (outcome.outcome !== "busy") result[outcome.outcome]++;+ if (outcome.outcome === "sent") await sleep(PACE_MS);+ }+ }++ await finishPromotions(touchedPromotions);+ if (refused.size > 0) result.retryAfterSeconds = Math.min(...refused.values());+ return result;+}++// A promotion with nothing left to send is sent.+async function finishPromotions(promotionIds: Set<string>): Promise<void> {+ for (const promotionId of promotionIds) {+ const owed = await prisma.emailSend.count({ where: { promotionId, state: { in: ["queued", "sending"] } } });+ if (owed > 0) continue;+ await prisma.promotion.updateMany({+ data: { finishedAt: new Date(), status: "sent" },+ where: { id: promotionId, status: "sending" },+ });+ }+}++export type OutboxKindRow = {+ kind: EmailKindId;+ priority: number;+ quota: EmailSendKind;+ queued: number;+ sentToday: number;+ failed: number;+};++export type OutboxItem = {+ id: string;+ kind: string;+ state: string;+ priority: number;+ rank: number;+ email: string | null;+ promotionName: string | null;+ attempts: number;+ error: string | null;+ notBefore: string;+ // Whether a failed attempt is holding the row back right now.+ backingOff: boolean;+ sentAt: string | null;+ createdAt: string;+};++export type OutboxOverview = {+ quota: DailyEmailQuotaStatus;+ kinds: OutboxKindRow[];+ // What is waiting, then what failed, then what went most recently.+ items: OutboxItem[];+ drainPending: boolean;+};++const ITEMS = 60;++/** The outbox as su sees it: the day's quota, every kind's standing, and the+ * rows worth a look. */+export async function outboxOverview(now = new Date()): Promise<OutboxOverview> {+ const dayStart = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));+ const [quota, priorities, queued, sentToday, failed, waiting, broken, recent, drain] = await Promise.all([+ dailyEmailQuotaStatus(now),+ getGlobalSetting("emailPriorities"),+ prisma.emailSend.groupBy({ _count: true, by: ["kind"], where: { state: { in: ["queued", "sending"] } } }),+ prisma.emailSend.groupBy({ _count: true, by: ["kind"], where: { sentAt: { gte: dayStart }, state: "sent" } }),+ prisma.emailSend.groupBy({ _count: true, by: ["kind"], where: { state: "failed" } }),+ prisma.emailSend.findMany({+ include: { promotion: { select: { name: true } }, user: { select: { email: true } } },+ orderBy: [{ priority: "desc" }, { rank: "asc" }, { createdAt: "asc" }],+ take: ITEMS,+ where: { state: { in: ["queued", "sending"] } },+ }),+ prisma.emailSend.findMany({+ include: { promotion: { select: { name: true } }, user: { select: { email: true } } },+ orderBy: { updatedAt: "desc" },+ take: ITEMS,+ where: { state: "failed" },+ }),+ prisma.emailSend.findMany({+ include: { promotion: { select: { name: true } }, user: { select: { email: true } } },+ orderBy: { sentAt: "desc" },+ take: ITEMS,+ where: { state: { in: ["sent", "skipped"] } },+ }),+ prisma.asyncJob.findFirst({ select: { id: true }, where: { kind: "email-drain", state: { in: ["queued", "running"] } } }),+ ]);+ const countOf = (groups: { kind: string; _count: number }[], kind: string) =>+ groups.find((g) => g.kind === kind)?._count ?? 0;+ const kinds = EMAIL_KIND_IDS.map((kind) => ({+ kind,+ priority: priorities[kind],+ quota: EMAIL_KINDS[kind].quota,+ queued: countOf(queued, kind),+ sentToday: countOf(sentToday, kind),+ failed: countOf(failed, kind),+ }));+ const items = [...waiting, ...broken, ...recent].map((row) => ({+ id: row.id,+ kind: row.kind,+ state: row.state,+ priority: row.priority,+ rank: row.rank,+ email: row.user?.email ?? null,+ promotionName: row.promotion?.name ?? null,+ attempts: row.attempts,+ error: row.error,+ notBefore: row.notBefore.toISOString(),+ backingOff: row.state === "queued" && row.notBefore.getTime() > now.getTime(),+ sentAt: row.sentAt?.toISOString() ?? null,+ createdAt: row.createdAt.toISOString(),+ }));+ return { quota, kinds, items, drainPending: drain !== null };+}++/** Puts a failed row back in the queue with a clean slate and sends for a+ * drainer. */+export async function retryEmail(id: string): Promise<boolean> {+ const { count } = await prisma.emailSend.updateMany({+ data: { attempts: 0, error: null, notBefore: new Date(), state: "queued" },+ where: { id, state: "failed" },+ });+ if (count > 0) await scheduleDrain(0);+ return count > 0;+}diff --git a/site/src/lib/email/resend.ts b/site/src/lib/email/resend.tsindex f944fb9b..e555d5d3 100644--- a/site/src/lib/email/resend.ts+++ b/site/src/lib/email/resend.ts@@ -23,6 +23,9 @@ export function registeredUsersSegmentId(): string {}// What every email path needs to know about an account.+/** One message as the provider takes it. */+export type EmailMessage = Parameters<Resend["emails"]["send"]>[0];+export type EmailUser = {id: string;email: string;diff --git a/site/src/lib/email/send-credits-expiring.ts b/site/src/lib/email/send-credits-expiring.tsindex 6449a963..48895409 100644--- a/site/src/lib/email/send-credits-expiring.ts+++ b/site/src/lib/email/send-credits-expiring.ts@@ -1,69 +1,44 @@import { DONKEYCUT_CANONICAL } from "@/cut/lib/hosts";-import {- DONKEY_LOGO_CID,- DONKEY_LOGO_PNG_BASE64,-} from "@/emails/_components/logo";+import { DONKEY_LOGO_CID, DONKEY_LOGO_PNG_BASE64 } from "@/emails/_components/logo";import CreditsExpiringEmail from "@/emails/credits-expiring";import { formatUsdPlain } from "@/lib/credits/format-usd";import { formatCreditExpiry } from "@/lib/credits/top-up";-import {- bulkFrom,- emailFrom,- getResend,- isResendConfigured,- ResendNotConfiguredError,- type EmailUser,-} from "@/lib/email/resend";-import {- unsubscribeActionUrl,- unsubscribePageUrl,-} from "@/lib/email/unsubscribe";+import { bulkFrom, emailFrom, type EmailMessage, type EmailUser } from "@/lib/email/resend";+import { unsubscribeActionUrl, unsubscribePageUrl } from "@/lib/email/unsubscribe";++export function creditsExpiringIdempotencyKey(grantId: string): string {+ return `credits-expiring:${grantId}`;+}// Tells one account what is left of a grant and the day it expires.-// `credits` is the remaining amount as a decimal string. The caller decides-// who gets it, records that it went, and names the send: one key per grant,-// so a retry within 24 hours cannot send twice and two grants on one account-// each get their own mail.-export async function sendCreditsExpiringEmail(- user: EmailUser,- credits: string,- expiresAt: Date,- idempotencyKey: string,-): Promise<void> {- if (!isResendConfigured()) throw new ResendNotConfiguredError();+// `credits` is the remaining amount as a decimal string.+export function buildCreditsExpiringEmail(user: EmailUser, credits: string, expiresAt: Date): EmailMessage {const from = bulkFrom();if (!from) throw new Error("No bulk sender configured.");-const firstName = user.name.trim().split(/\s+/)[0] || user.name;- const { error } = await getResend().emails.send(- {- from,- to: user.email,- replyTo: emailFrom() || from,- subject: `Your ${formatUsdPlain(credits)} in AI credits expires ${formatCreditExpiry(expiresAt)}`,- react: CreditsExpiringEmail({- credits: formatUsdPlain(credits),- editorUrl: `${DONKEYCUT_CANONICAL}/app`,- expiresOn: formatCreditExpiry(expiresAt),- name: firstName,- unsubscribeUrl: unsubscribePageUrl(user.id),- }),- attachments: [- {- content: DONKEY_LOGO_PNG_BASE64,- contentId: DONKEY_LOGO_CID,- contentType: "image/png",- filename: "donkey-cut.png",- },- ],- headers: {- "List-Unsubscribe": `<${unsubscribeActionUrl(user.id)}>`,- "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",+ return {+ from,+ to: user.email,+ replyTo: emailFrom() || from,+ subject: `Your ${formatUsdPlain(credits)} in AI credits expires ${formatCreditExpiry(expiresAt)}`,+ react: CreditsExpiringEmail({+ credits: formatUsdPlain(credits),+ editorUrl: `${DONKEYCUT_CANONICAL}/app`,+ expiresOn: formatCreditExpiry(expiresAt),+ name: firstName,+ unsubscribeUrl: unsubscribePageUrl(user.id),+ }),+ attachments: [+ {+ content: DONKEY_LOGO_PNG_BASE64,+ contentId: DONKEY_LOGO_CID,+ contentType: "image/png",+ filename: "donkey-cut.png",},+ ],+ headers: {+ "List-Unsubscribe": `<${unsubscribeActionUrl(user.id)}>`,+ "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",},- { idempotencyKey },- );- if (error) {- throw new Error(`Resend send failed: ${error.name}: ${error.message}`);- }+ };}diff --git a/site/src/lib/email/send-credits-offered.ts b/site/src/lib/email/send-credits-offered.tsindex 5ef50379..5a8ab478 100644--- a/site/src/lib/email/send-credits-offered.ts+++ b/site/src/lib/email/send-credits-offered.ts@@ -1,23 +1,10 @@-import {- DONKEY_LOGO_CID,- DONKEY_LOGO_PNG_BASE64,-} from "@/emails/_components/logo";+import { DONKEY_LOGO_CID, DONKEY_LOGO_PNG_BASE64 } from "@/emails/_components/logo";import CreditsOfferedEmail from "@/emails/credits-offered";import { creditMicrosToString } from "@/lib/credits/amounts";import { formatUsdPlain } from "@/lib/credits/format-usd";import { formatCreditExpiry } from "@/lib/credits/top-up";-import {- bulkFrom,- emailFrom,- getResend,- isResendConfigured,- ResendNotConfiguredError,- type EmailUser,-} from "@/lib/email/resend";-import {- unsubscribeActionUrl,- unsubscribePageUrl,-} from "@/lib/email/unsubscribe";+import { bulkFrom, emailFrom, type EmailMessage, type EmailUser } from "@/lib/email/resend";+import { unsubscribeActionUrl, unsubscribePageUrl } from "@/lib/email/unsubscribe";// How long a claimed offer lives, as people read it.export function describeCreditLifetime(days: number | null): string | null {@@ -28,49 +15,45 @@ export function describeCreditLifetime(days: number | null): string | null {return days === 1 ? "a day" : `${days} days`;}+export function creditsOfferedIdempotencyKey(offerId: string): string {+ return `credits-offered:${offerId}`;+}+// Tells the account credit is waiting and hands it the claim link. Credit// given by hand is account mail, so it goes whatever the marketing choice;// the footer still offers the opt-out.-export async function sendCreditsOfferedEmail(+export function buildCreditsOfferedEmail(user: EmailUser,- offer: { amountMicros: bigint; claimUrl: string; closesAt: Date; expiresAfterDays: number | null; id: string },-): Promise<void> {- if (!isResendConfigured()) throw new ResendNotConfiguredError();+ offer: { amountMicros: bigint; claimUrl: string; closesAt: Date; expiresAfterDays: number | null },+): EmailMessage {const from = bulkFrom();if (!from) throw new Error("No bulk sender configured.");-const credits = formatUsdPlain(creditMicrosToString(offer.amountMicros));const firstName = user.name.trim().split(/\s+/)[0] || user.name;- const { error } = await getResend().emails.send(- {- from,- to: user.email,- replyTo: emailFrom() || from,- subject: `${credits} in AI credits is waiting for you`,- react: CreditsOfferedEmail({- claimBy: formatCreditExpiry(offer.closesAt),- claimUrl: offer.claimUrl,- credits,- lifetime: describeCreditLifetime(offer.expiresAfterDays),- name: firstName,- unsubscribeUrl: unsubscribePageUrl(user.id),- }),- attachments: [- {- content: DONKEY_LOGO_PNG_BASE64,- contentId: DONKEY_LOGO_CID,- contentType: "image/png",- filename: "donkey-cut.png",- },- ],- headers: {- "List-Unsubscribe": `<${unsubscribeActionUrl(user.id)}>`,- "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",+ return {+ from,+ to: user.email,+ replyTo: emailFrom() || from,+ subject: `${credits} in AI credits is waiting for you`,+ react: CreditsOfferedEmail({+ claimBy: formatCreditExpiry(offer.closesAt),+ claimUrl: offer.claimUrl,+ credits,+ lifetime: describeCreditLifetime(offer.expiresAfterDays),+ name: firstName,+ unsubscribeUrl: unsubscribePageUrl(user.id),+ }),+ attachments: [+ {+ content: DONKEY_LOGO_PNG_BASE64,+ contentId: DONKEY_LOGO_CID,+ contentType: "image/png",+ filename: "donkey-cut.png",},+ ],+ headers: {+ "List-Unsubscribe": `<${unsubscribeActionUrl(user.id)}>`,+ "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",},- { idempotencyKey: `credits-offered:${offer.id}` },- );- if (error) {- throw new Error(`Resend send failed: ${error.name}: ${error.message}`);- }+ };}diff --git a/site/src/lib/email/send-welcome.ts b/site/src/lib/email/send-welcome.tsindex 0e1b5be0..bc71cbfa 100644--- a/site/src/lib/email/send-welcome.ts+++ b/site/src/lib/email/send-welcome.ts@@ -1,100 +1,58 @@-import {- DONKEY_LOGO_CID,- DONKEY_LOGO_PNG_BASE64,-} from "@/emails/_components/logo";+import { DONKEY_LOGO_CID, DONKEY_LOGO_PNG_BASE64 } from "@/emails/_components/logo";import WelcomeEmail from "@/emails/welcome";-import {- bulkFrom,- emailFrom,- getResend,- isResendConfigured,- type EmailUser,-} from "@/lib/email/resend";-import {- unsubscribeActionUrl,- unsubscribePageUrl,-} from "@/lib/email/unsubscribe";+import { bulkFrom, emailFrom, type EmailMessage, type EmailUser } from "@/lib/email/resend";+import { unsubscribeActionUrl, unsubscribePageUrl } from "@/lib/email/unsubscribe";import { prisma } from "@/lib/prisma";-// Sends the one-time welcome email. Two layers keep it single-send: the row-// claim below is the permanent record (only the caller whose conditional-// update flips the null actually sends), and Resend's idempotency key covers a-// crash between claim release and retry for 24 hours.+// The one-time welcome email. The outbox row keyed on the account is what+// keeps it single-send; the settings row records that it went.// `credits` is the amount the signup grant landed, or null when the address-// had already received it on an earlier account; the email names it only when-// it is there.-export async function sendWelcomeEmail(- user: EmailUser,- credits: string | null,-): Promise<void> {- if (!isResendConfigured()) {- console.log("[email] RESEND_API_KEY not set; skipping welcome email", {- userId: user.id,- });- return;- }- const from = bulkFrom();- if (!from) {- console.log("[email] no sender configured; skipping welcome email", {- userId: user.id,- });- return;- }-- await prisma.userEmailSettings.upsert({- create: { userId: user.id },- update: {},- where: { userId: user.id },- });- const claim = await prisma.userEmailSettings.updateMany({- data: { welcomeEmailSentAt: new Date() },- where: { userId: user.id, welcomeEmailSentAt: null },- });- if (claim.count === 0) {- return;- }+// had already received it on an earlier account; the email names it only+// when it is there.+export function welcomeIdempotencyKey(userId: string): string {+ return `welcome-email:${userId}`;+}+export function buildWelcomeEmail(user: EmailUser, credits: string | null): EmailMessage {+ const from = bulkFrom();+ if (!from) throw new Error("No bulk sender configured.");const firstName = user.name.trim().split(/\s+/)[0] || user.name;- const { error } = await getResend().emails.send(- {- from,- to: user.email,- // The bulk subdomain sends but does not receive; a reply goes to the- // apex address, where Google Workspace delivers it.- replyTo: emailFrom() || from,- subject: "Thanks for signing up ❤️",- react: WelcomeEmail({- credits,- name: firstName,- unsubscribeUrl: unsubscribePageUrl(user.id),- }),- // The mark rides inside the message (Content-ID embed) so no client- // fetches anything external; Gmail strips data URIs, so cid is the only- // fetch-free form that renders everywhere.- attachments: [- {- content: DONKEY_LOGO_PNG_BASE64,- contentId: DONKEY_LOGO_CID,- contentType: "image/png",- filename: "donkey-cut.png",- },- ],- // RFC 8058 one-click: mail providers POST here with no session; the- // signed token in the URL is the authorization.- headers: {- "List-Unsubscribe": `<${unsubscribeActionUrl(user.id)}>`,- "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",+ return {+ from,+ to: user.email,+ // The bulk subdomain sends but does not receive; a reply goes to the+ // apex address, where Google Workspace delivers it.+ replyTo: emailFrom() || from,+ subject: "Thanks for signing up ❤️",+ react: WelcomeEmail({+ credits,+ name: firstName,+ unsubscribeUrl: unsubscribePageUrl(user.id),+ }),+ // The mark rides inside the message (Content-ID embed) so no client+ // fetches anything external; Gmail strips data URIs, so cid is the only+ // fetch-free form that renders everywhere.+ attachments: [+ {+ content: DONKEY_LOGO_PNG_BASE64,+ contentId: DONKEY_LOGO_CID,+ contentType: "image/png",+ filename: "donkey-cut.png",},+ ],+ // RFC 8058 one-click: mail providers POST here with no session; the+ // signed token in the URL is the authorization.+ headers: {+ "List-Unsubscribe": `<${unsubscribeActionUrl(user.id)}>`,+ "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",},- { idempotencyKey: `welcome-email:${user.id}` },- );- if (error) {- // Release the claim so the next provisioning run can retry; the- // idempotency key keeps a near-simultaneous retry from double-sending.- await prisma.userEmailSettings.updateMany({- data: { welcomeEmailSentAt: null },- where: { userId: user.id },- });- throw new Error(`Resend send failed: ${error.name}: ${error.message}`);- }+ };+}++export async function recordWelcomeSent(userId: string): Promise<void> {+ await prisma.userEmailSettings.upsert({+ create: { userId, welcomeEmailSentAt: new Date() },+ update: { welcomeEmailSentAt: new Date() },+ where: { userId },+ });}diff --git a/site/src/lib/jobs/credit-expiry-notice.ts b/site/src/lib/jobs/credit-expiry-notice.tsindex 9def959d..6adc425e 100644--- a/site/src/lib/jobs/credit-expiry-notice.ts+++ b/site/src/lib/jobs/credit-expiry-notice.ts@@ -1,21 +1,17 @@import { z } from "zod";import { getGlobalSetting } from "@/lib/config/effective";-import { sendDueCreditExpiryNotices } from "@/lib/email/credit-expiry-notices";-import { ResendNotConfiguredError } from "@/lib/email/resend";-import { defineJob, JobFailure } from "@/lib/jobs/registry";+import { queueDueCreditExpiryNotices } from "@/lib/email/credit-expiry-notices";+import { drainAsJob } from "@/lib/jobs/email-drain";+import { defineJob } from "@/lib/jobs/registry";// The daily credit-expiry notice. Every account holding signup or manually-// granted credit that expires within the creditExpiryNotice window gets one-// email naming what is left and the day it goes. Runs from-// /api/credits/expiry-notices.+// granted credit that expires within the creditExpiryNotice window is queued+// one email naming what is left and the day it goes, and the outbox is+// drained. Runs from /api/credits/expiry-notices.export const creditExpiryNoticeJob = defineJob(z.object({}).strict(), async () => {const setting = await getGlobalSetting("creditExpiryNotice");- try {- return await sendDueCreditExpiryNotices({ withinDays: setting.daysBefore });- } catch (e) {- // Permanent: an unconfigured sender never clears on retry.- if (e instanceof ResendNotConfiguredError) throw new JobFailure(e.message);- throw e;- }+ const queued = await queueDueCreditExpiryNotices({ withinDays: setting.daysBefore });+ const drained = await drainAsJob();+ return { ...queued, drained };});diff --git a/site/src/lib/jobs/email-drain.ts b/site/src/lib/jobs/email-drain.tsnew file mode 100644index 00000000..adeffea8--- /dev/null+++ b/site/src/lib/jobs/email-drain.ts@@ -0,0 +1,26 @@+import { z } from "zod";++import { drainOutbox, type DrainResult } from "@/lib/email/outbox";+import { defineJob, JobDeferred } from "@/lib/jobs/registry";++// Under the worker's 300 s maxDuration, with room for the continuation's+// own writes.+const BUDGET_MS = 200_000;++/** Runs the drainer inside a job: a run that ran out of time queues its+ * continuation, and one that ran out of quota comes back when the quota is+ * expected to open. */+export async function drainAsJob(): Promise<DrainResult> {+ const result = await drainOutbox(BUDGET_MS);+ if (result.more) {+ const { enqueueJob } = await import("@/lib/jobs/queue");+ await enqueueJob("email-drain", {}, "system");+ return result;+ }+ if (result.retryAfterSeconds !== null) {+ throw new JobDeferred("Daily email quota reached; the outbox resumes when it opens.", result.retryAfterSeconds);+ }+ return result;+}++export const emailDrainJob = defineJob(z.object({}).strict(), async () => drainAsJob());diff --git a/site/src/lib/jobs/outreach-scan.ts b/site/src/lib/jobs/outreach-scan.tsindex bd57cb57..be874a1e 100644--- a/site/src/lib/jobs/outreach-scan.ts+++ b/site/src/lib/jobs/outreach-scan.ts@@ -25,6 +25,7 @@ import {OUTREACH_STORAGE_FULL_SHARE,type OutreachReason,} from "@/lib/marketing/campaigns";+import { lastActiveByUser } from "@/lib/marketing/lastActive";import { prisma } from "@/lib/prisma";const DAY_MS = 24 * 60 * 60 * 1000;@@ -218,26 +219,8 @@ async function scan() {return { added: 0, dropped: await retire(), scanned: 0 };}- const [billed, uploaded, edited, stored, ranOut, existing] = await Promise.all([- prisma.inferenceUsageEvent.groupBy({- _max: { createdAt: true },- by: ["userId"],- where: { billingStatus: "charged", userId: { in: userIds } },- }),- // Cloud work the person did by hand: media they put up and projects they- // edited. Housekeeping writes the storage counter on its own — GC sweeps,- // copy jobs, the render worker — so its timestamp says nothing about a- // person being there.- prisma.cutMediaObject.groupBy({- _max: { createdAt: true },- by: ["userId"],- where: { quotaExempt: false, uploadState: "complete", userId: { in: userIds } },- }),- prisma.cutProject.groupBy({- _max: { updatedAt: true },- by: ["userId"],- where: { userId: { in: userIds } },- }),+ const [lastActiveBy, stored, ranOut, existing] = await Promise.all([+ lastActiveByUser(userIds),prisma.cutStorageUsage.findMany({select: { bytes: true, userId: true },where: { userId: { in: userIds } },@@ -255,16 +238,6 @@ async function scan() {}),]);- // Model work and cloud editing are both use of the product, so either one- // keeps an account looking as recent as it is.- const lastActiveBy = new Map<string, Date>();- const touch = (userId: string, at: Date | null) => {- const best = newest(at, lastActiveBy.get(userId));- if (best) lastActiveBy.set(userId, best);- };- for (const row of billed) touch(row.userId, row._max.createdAt);- for (const row of uploaded) touch(row.userId, row._max.createdAt);- for (const row of edited) touch(row.userId, row._max.updatedAt);const storedBy = new Map(stored.map((row) => [row.userId, row.bytes]));const ranOutBy = new Map(ranOut.map((row) => [row.userId, row._max.createdAt]));const existingBy = new Map(existing.map((row) => [row.userId, row]));diff --git a/site/src/lib/jobs/promotion-send.ts b/site/src/lib/jobs/promotion-send.tsdeleted file mode 100644index 2c42cb79..00000000--- a/site/src/lib/jobs/promotion-send.ts+++ /dev/null@@ -1,102 +0,0 @@-import { z } from "zod";--import { ResendNotConfiguredError } from "@/lib/email/resend";-import { isMarketingUnsubscribed } from "@/lib/email/unsubscribe";-import { defineJob, JobFailure } from "@/lib/jobs/registry";-import { UnknownPlaceholderError } from "@/lib/marketing/placeholders";-import { MAX_SEND_ATTEMPTS } from "@/lib/marketing/promotionInput";-import { promotionCopyOf, sendPromotionEmail } from "@/lib/marketing/promotions";-import { prisma } from "@/lib/prisma";--// Sends a promotion to the recipient rows its send resolved. One email at a-// time, paced under the provider's rate limit, each keyed on the recipient so-// a re-run mails nobody twice. The opt-out is read again right before each-// send, so someone who unsubscribed after the segment was resolved is-// skipped. A failed send is tried again later in the run, up to MAX_SEND_ATTEMPTS,-// with a pause after each failure so a provider blip does not burn the list.-// A run stops inside the worker's time budget and queues its own-// continuation; the last run marks the promotion sent.--const TAKE = 50;-const PACE_MS = 550;-const FAILURE_PAUSE_MS = 5_000;-// Under the worker's 300 s maxDuration, with room for the pause and the-// continuation's own writes.-const BUDGET_MS = 200_000;--const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));--const payloadSchema = z.object({ promotionId: z.string().min(1) }).strict();--export const promotionSendJob = defineJob(payloadSchema, async ({ promotionId }) => {- const promotion = await prisma.promotion.findUnique({ where: { id: promotionId } });- if (!promotion) throw new JobFailure("The promotion no longer exists.");- if (promotion.status !== "sending") return { promotionId, skipped: promotion.status };- const copy = promotionCopyOf(promotion);-- const startedAt = Date.now();- let sent = 0;- let failed = 0;- let skipped = 0;- const continueLater = async () => {- const { enqueueJob } = await import("@/lib/jobs/queue");- await enqueueJob("promotion-send", { promotionId }, promotion.actorUserId ?? "system");- return { continued: true, failed, promotionId, sent, skipped };- };-- for (;;) {- // Rows still owed a send: never sent, not given up on. A row that failed- // earlier in this run comes back at the end of the order.- const pending = await prisma.promotionRecipient.findMany({- orderBy: [{ attempts: "asc" }, { createdAt: "asc" }],- select: { attempts: true, id: true, user: { select: { email: true, id: true, name: true } } },- take: TAKE,- where: { attempts: { lt: MAX_SEND_ATTEMPTS }, promotionId, sentAt: null },- });- if (pending.length === 0) break;- for (const recipient of pending) {- if (Date.now() - startedAt > BUDGET_MS) return continueLater();-- if (await isMarketingUnsubscribed(recipient.user.id)) {- await prisma.promotionRecipient.update({- data: { attempts: MAX_SEND_ATTEMPTS, error: "Unsubscribed before the send." },- where: { id: recipient.id },- });- skipped++;- continue;- }- try {- await sendPromotionEmail({- copy,- idempotencyKey: `promotion:${promotionId}:${recipient.user.id}`,- user: recipient.user,- });- await prisma.promotionRecipient.update({- data: { error: null, sentAt: new Date() },- where: { id: recipient.id },- });- sent++;- await sleep(PACE_MS);- } catch (error) {- // Permanent for the whole send: the draft comes back so the operator- // can fix it, and the unsent rows go with it.- if (error instanceof ResendNotConfiguredError || error instanceof UnknownPlaceholderError) {- await prisma.promotionRecipient.deleteMany({ where: { promotionId, sentAt: null } });- await prisma.promotion.update({ data: { status: "draft" }, where: { id: promotionId } });- throw new JobFailure(error.message);- }- await prisma.promotionRecipient.update({- data: {- attempts: { increment: 1 },- error: error instanceof Error ? error.message : String(error),- },- where: { id: recipient.id },- });- if (recipient.attempts + 1 >= MAX_SEND_ATTEMPTS) failed++;- await sleep(FAILURE_PAUSE_MS);- }- }- }- await prisma.promotion.update({ data: { finishedAt: new Date(), status: "sent" }, where: { id: promotionId } });- return { failed, promotionId, sent, skipped };-});diff --git a/site/src/lib/jobs/queue.ts b/site/src/lib/jobs/queue.tsindex 43fff9c0..a721228e 100644--- a/site/src/lib/jobs/queue.ts+++ b/site/src/lib/jobs/queue.ts@@ -7,7 +7,7 @@import { NextResponse } from "next/server";import type { Prisma } from "@/generated/prisma/client";-import { JobFailure, jobKinds } from "@/lib/jobs/registry";+import { JobDeferred, JobFailure, jobKinds } from "@/lib/jobs/registry";import { prisma } from "@/lib/prisma";const JOBS_QUEUE_NAME = "donkey-jobs";@@ -19,6 +19,8 @@ const STALE_QUEUED_MS = 30_000;// claim treats such a row as reclaimable so a queue retry can run it again,// and the status poll surfaces it as an error instead of spinning forever.const STALE_RUNNING_MS = 15 * 60_000;+// The longest the queue will hold a message before delivery.+const MAX_QUEUE_DELAY_S = 12 * 60 * 60;type JobRow = NonNullable<Awaited<ReturnType<typeof prisma.asyncJob.findUnique>>>;@@ -43,8 +45,9 @@ async function queueId(accountId: string, token: string): Promise<string | null>return cachedQueueId;}-/** Publish one job id to the jobs queue; false when unconfigured or failed. */-async function publishJob(jobId: string): Promise<boolean> {+/** Publish one job id to the jobs queue, held back by the delay; false when+ * unconfigured or failed. */+async function publishJob(jobId: string, delaySeconds = 0): Promise<boolean> {const accountId = process.env.R2_ACCOUNT_ID;const token = process.env.CLOUDFLARE_QUEUES_API_TOKEN;if (!accountId || !token) return false;@@ -56,7 +59,11 @@ async function publishJob(jobId: string): Promise<boolean> {{method: "POST",headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },- body: JSON.stringify({ body: { jobId }, content_type: "json" }),+ body: JSON.stringify({+ body: { jobId },+ content_type: "json",+ delay_seconds: Math.min(delaySeconds, MAX_QUEUE_DELAY_S),+ }),},);return res.ok;@@ -65,16 +72,17 @@ async function publishJob(jobId: string): Promise<boolean> {}}-/** Queue the job and hand back its id; without a queue (local dev) the job- * runs now, before responding — a serverless response must not leave- * fire-and-forget work behind. */+/** Queue the job, held back by the delay, and hand back its id; without a+ * queue (local dev) the job runs now, before responding — a serverless+ * response must not leave fire-and-forget work behind. */export async function enqueueJob(kind: string,payload: Prisma.InputJsonValue,createdBy: string,+ options: { delaySeconds?: number } = {},): Promise<{ jobId: string }> {const job = await prisma.asyncJob.create({ data: { kind, payload, createdBy } });- const queued = await publishJob(job.id);+ const queued = await publishJob(job.id, options.delaySeconds);if (!queued) await executeJob(job.id);return { jobId: job.id };}@@ -109,7 +117,9 @@ export async function jobStatusResponse(job: JobRow): Promise<Response> {/** Execute one job (called by the queue consumer through /api/jobs/worker).* A 200 consumes the message (including permanent failures, which land on the- * job row); a 500 releases the claim and lets the queue retry. */+ * job row); a 500 releases the claim and lets the queue retry. A deferral+ * consumes the message too and publishes a fresh one held back by the job's+ * own delay, so waiting on a quota never spends the queue's retries. */export async function executeJob(jobId: string): Promise<Response> {// A stale running row is a killed execution, so the claim takes it too —// that way the queue's redelivery actually re-runs it instead of ack'ing.@@ -150,6 +160,16 @@ export async function executeJob(jobId: string): Promise<Response> {});return NextResponse.json({ ok: true });} catch (e) {+ if (e instanceof JobDeferred) {+ await prisma.asyncJob+ .updateMany({ where: { id: jobId, state: "running" }, data: { state: "queued" } })+ .catch(() => {});+ if (await publishJob(jobId, e.retryAfterSeconds)) return NextResponse.json({ ok: true });+ return NextResponse.json(+ { error: e.message },+ { headers: { "Retry-After": String(e.retryAfterSeconds) }, status: 503 },+ );+ }if (e instanceof JobFailure) return fail(e.message);// Transient failure: release the claim so the queue's retry can run the// job again from where its idempotent steps left off.diff --git a/site/src/lib/jobs/registry.ts b/site/src/lib/jobs/registry.tsindex 653a408c..972dfab4 100644--- a/site/src/lib/jobs/registry.ts+++ b/site/src/lib/jobs/registry.ts@@ -8,13 +8,23 @@ import { creditExpiryNoticeJob } from "@/lib/jobs/credit-expiry-notice";import { deleteUserJob } from "@/lib/jobs/delete-user";import { experimentResultsJob } from "@/lib/jobs/experiment-results";import { outreachScanJob } from "@/lib/jobs/outreach-scan";-import { promotionSendJob } from "@/lib/jobs/promotion-send";+import { emailDrainJob } from "@/lib/jobs/email-drain";// Thrown by an executor when the job can never succeed — the message lands on// the job row as its error. Anything else thrown is transient: the claim is// released and the queue redelivers.export class JobFailure extends Error {}+export class JobDeferred extends Error {+ public constructor(+ message: string,+ public readonly retryAfterSeconds: number,+ ) {+ super(message);+ this.name = "JobDeferred";+ }+}+export type JobKind = {// Validates a payload at creation time and again at execution time; a stored// payload that no longer parses is a permanent failure.@@ -35,7 +45,7 @@ export const jobKinds: Record<string, JobKind> = {"credit-expiry": creditExpiryJob,"credit-expiry-notice": creditExpiryNoticeJob,"delete-user": deleteUserJob,+ "email-drain": emailDrainJob,"experiment-results": experimentResultsJob,"outreach-scan": outreachScanJob,- "promotion-send": promotionSendJob,};diff --git a/site/src/lib/marketing/forward-reply.ts b/site/src/lib/marketing/forward-reply.tsindex 82e7af2c..1729bbb9 100644--- a/site/src/lib/marketing/forward-reply.ts+++ b/site/src/lib/marketing/forward-reply.ts@@ -1,4 +1,7 @@-import { emailFrom, getResend, isResendConfigured } from "@/lib/email/resend";+import { z } from "zod";++import { deliverEmail } from "@/lib/email/outbox";+import { emailFrom, getResend, isResendConfigured, type EmailMessage } from "@/lib/email/resend";import { outreachIdFromRecipients } from "@/lib/marketing/replyAddress";import { prisma } from "@/lib/prisma";@@ -27,27 +30,27 @@ type ReceivedEmail = {subject: string;};+export const replyForwardPayloadSchema = z.object({ emailId: z.string().min(1), outreachId: z.string().min(1) }).strict();+export type ReplyForwardPayload = z.output<typeof replyForwardPayloadSchema>;+// Handles an inbound reply to a note that carried the signed reply alias —-// one sent with reply tracking on. Flips the row to replied and-// forwards the message on so it can be answered from a normal inbox. The-// forward goes out as a fresh send rather than Resend's own forward call-// because only a send takes a reply-to — that is what makes hitting reply in-// the inbox address the user instead of looping back into the reply domain.+// one sent with reply tracking on. Flips the row to replied and queues the+// forward so it can be answered from a normal inbox. The forward goes out as+// a fresh send rather than Resend's own forward call because only a send+// takes a reply-to — that is what makes hitting reply in the inbox address+// the user instead of looping back into the reply domain.//-// Nothing about the message is written down. Body, subject, and attachments-// travel from Resend through this function to the inbox and stop there.+// Nothing about the message is written down beyond its id. Body, subject, and+// attachments travel from Resend through the builder to the inbox and stop+// there.export async function forwardOutreachReply(email: ReceivedEmail): Promise<boolean> {const outreachId = outreachIdFromRecipients(email.received_for, email.to);- if (!outreachId) {- return false;- }+ if (!outreachId) return false;const outreach = await prisma.userOutreach.findUnique({- select: { id: true, repliedAt: true, user: { select: { email: true, name: true } } },+ select: { id: true, repliedAt: true },where: { id: outreachId },});- if (!outreach) {- return false;- }+ if (!outreach) return false;// Keep the first reply's timestamp; a later one in the same thread only// confirms the status.@@ -56,33 +59,41 @@ export async function forwardOutreachReply(email: ReceivedEmail): Promise<booleawhere: { id: outreach.id },});- const inbox = outreachInbox();- const from = emailFrom();- if (!isResendConfigured() || !inbox || !from) {- console.warn("[outreach] reply not forwarded; inbox or sender unconfigured", {- outreachId: outreach.id,- });+ if (!isResendConfigured() || !outreachInbox() || !emailFrom()) {+ console.warn("[outreach] reply not forwarded; inbox or sender unconfigured", { outreachId: outreach.id });return true;}+ await deliverEmail({+ // Resend's own dedupe is what keeps a redelivered webhook from forwarding+ // the same reply twice, and the outbox key mirrors it.+ idempotencyKey: `outreach-reply:${email.email_id}`,+ kind: "reply-forward",+ payload: { emailId: email.email_id, outreachId: outreach.id },+ });+ return true;+}++/** Fetches the reply from Resend and wraps it for the operator's inbox. */+export async function buildReplyForwardEmail(payload: ReplyForwardPayload): Promise<EmailMessage> {+ const inbox = outreachInbox();+ const from = emailFrom();+ if (!inbox || !from) throw new Error("RESEND_INBOX_EMAIL or RESEND_FROM_EMAIL is not configured.");+ const outreach = await prisma.userOutreach.findUnique({+ select: { user: { select: { email: true, name: true } } },+ where: { id: payload.outreachId },+ });+ if (!outreach) throw new Error("The outreach row no longer exists.");const resend = getResend();- const received = await resend.emails.receiving.get(email.email_id);+ const received = await resend.emails.receiving.get(payload.emailId);if (received.error || !received.data) {- throw new Error(- `Resend inbound fetch failed: ${received.error?.name}: ${received.error?.message}`,- );+ throw new Error(`Resend inbound fetch failed: ${received.error?.name}: ${received.error?.message}`);}-const attachments = await Promise.all(received.data.attachments.map(async (attachment) => {- const file = await resend.emails.receiving.attachments.get({- emailId: email.email_id,- id: attachment.id,- });+ const file = await resend.emails.receiving.attachments.get({ emailId: payload.emailId, id: attachment.id });if (file.error || !file.data) {- throw new Error(- `Resend attachment fetch failed: ${file.error?.name}: ${file.error?.message}`,- );+ throw new Error(`Resend attachment fetch failed: ${file.error?.name}: ${file.error?.message}`);}return {contentType: file.data.content_type,@@ -94,24 +105,13 @@ export async function forwardOutreachReply(email: ReceivedEmail): Promise<booleaconst banner = `${outreach.user.name} <${outreach.user.email}> replied to your outreach note.`;const bannerHtml = escapeHtml(banner);- const { error } = await resend.emails.send(- {- from,- to: inbox,- replyTo: outreach.user.email,- subject: received.data.subject || "Re: What are you making?",- attachments: attachments.length > 0 ? attachments : undefined,- html: received.data.html- ? `<p><strong>${bannerHtml}</strong></p>${received.data.html}`- : undefined,- text: received.data.text ? `${banner}\n\n${received.data.text}` : banner,- },- // Resend's own dedupe is what keeps a redelivered webhook from forwarding- // the same reply twice, so nothing has to be written down to track it.- { idempotencyKey: `outreach-reply:${email.email_id}` },- );- if (error) {- throw new Error(`Resend forward failed: ${error.name}: ${error.message}`);- }- return true;+ return {+ from,+ to: inbox,+ replyTo: outreach.user.email,+ subject: received.data.subject || "Re: What are you making?",+ attachments: attachments.length > 0 ? attachments : undefined,+ html: received.data.html ? `<p><strong>${bannerHtml}</strong></p>${received.data.html}` : undefined,+ text: received.data.text ? `${banner}\n\n${received.data.text}` : banner,+ };}diff --git a/site/src/lib/marketing/lastActive.ts b/site/src/lib/marketing/lastActive.tsnew file mode 100644index 00000000..6e5a7ee0--- /dev/null+++ b/site/src/lib/marketing/lastActive.ts@@ -0,0 +1,43 @@+import { prisma } from "@/lib/prisma";++// When each account last used the product. Model work and cloud editing are+// both use, so either one keeps an account looking as recent as it is.+// Housekeeping writes the storage counter on its own — GC sweeps, copy jobs,+// the render worker — so that timestamp says nothing about a person being+// there and is left out.+export async function lastActiveByUser(userIds: string[]): Promise<Map<string, Date>> {+ const lastActiveBy = new Map<string, Date>();+ if (userIds.length === 0) return lastActiveBy;+ const [billed, uploaded, edited] = await Promise.all([+ prisma.inferenceUsageEvent.groupBy({+ _max: { createdAt: true },+ by: ["userId"],+ where: { billingStatus: "charged", userId: { in: userIds } },+ }),+ prisma.cutMediaObject.groupBy({+ _max: { createdAt: true },+ by: ["userId"],+ where: { quotaExempt: false, uploadState: "complete", userId: { in: userIds } },+ }),+ prisma.cutProject.groupBy({+ _max: { updatedAt: true },+ by: ["userId"],+ where: { userId: { in: userIds } },+ }),+ ]);+ const touch = (userId: string, at: Date | null) => {+ const best = lastActiveBy.get(userId);+ if (at && (!best || at > best)) lastActiveBy.set(userId, at);+ };+ for (const row of billed) touch(row.userId, row._max.createdAt);+ for (const row of uploaded) touch(row.userId, row._max.createdAt);+ for (const row of edited) touch(row.userId, row._max.updatedAt);+ return lastActiveBy;+}++/** The ids most recently active first; accounts never seen using the+ * product come last, in the order given. */+export function byMostRecentlyActive<T extends { id: string }>(users: T[], lastActiveBy: Map<string, Date>): T[] {+ const at = (user: T) => lastActiveBy.get(user.id)?.getTime() ?? 0;+ return [...users].sort((a, b) => at(b) - at(a));+}diff --git a/site/src/lib/marketing/promotions.ts b/site/src/lib/marketing/promotions.tsindex cf4090ba..7c213d6d 100644--- a/site/src/lib/marketing/promotions.ts+++ b/site/src/lib/marketing/promotions.ts@@ -4,18 +4,11 @@ import type { Prisma } from "@/generated/prisma/client";import { DONKEY_LOGO_CID, DONKEY_LOGO_PNG_BASE64 } from "@/emails/_components/logo";import PromotionEmail from "@/emails/promotion";import { collectFacts } from "@/lib/config/audienceFacts";-import {- bulkFrom,- emailFrom,- getResend,- isResendConfigured,- ResendNotConfiguredError,- type EmailUser,-} from "@/lib/email/resend";+import { bulkFrom, emailFrom, type EmailMessage, type EmailUser } from "@/lib/email/resend";+import { PermanentSendError } from "@/lib/email/errors";import { unsubscribeActionUrl, unsubscribePageUrl } from "@/lib/email/unsubscribe";import { renderPromotion, type PromotionCopy } from "@/lib/marketing/promotionCopy";import {- MAX_SEND_ATTEMPTS,PROMOTION_SENDERS,PROMOTION_STATUSES,type PromotionSender,@@ -24,6 +17,7 @@ import {type SegmentCount,} from "@/lib/marketing/promotionInput";import { prisma } from "@/lib/prisma";+import { promotionOfferOf } from "@/lib/marketing/promotionOfferInput";// Promotions on the server: which address each sender is, who a segment// resolves to, one send, and the list su reads.@@ -62,47 +56,36 @@ export function promotionAudienceOf(raw: Prisma.JsonValue): Audience {return audienceSchema.parse(raw);}-/** Sends one promotion email. The idempotency key is the caller's: the job- * keys on the recipient so a retried run cannot mail anyone twice. */-export async function sendPromotionEmail(input: {- copy: PromotionCopy;- user: EmailUser;- idempotencyKey: string;-}): Promise<void> {- if (!isResendConfigured()) throw new ResendNotConfiguredError();- const from = promotionSenders()[input.copy.sender];- if (!from) throw new Error(`The ${input.copy.sender} sender is not configured.`);- const rendered = renderPromotion(input.copy, input.user);- const { error } = await getResend().emails.send(- {- from,- to: input.user.email,- replyTo: emailFrom() || from,- subject: rendered.subject,- react: PromotionEmail({- blocks: rendered.blocks,- cta: rendered.cta,- preview: rendered.preview,- unsubscribeUrl: unsubscribePageUrl(input.user.id),- }),- attachments: [- {- content: DONKEY_LOGO_PNG_BASE64,- contentId: DONKEY_LOGO_CID,- contentType: "image/png",- filename: "donkey-cut.png",- },- ],- headers: {- "List-Unsubscribe": `<${unsubscribeActionUrl(input.user.id)}>`,- "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",+/** One promotion email for one account. The idempotency key is the row's:+ * the outbox keys on the recipient so a retried run cannot mail anyone twice. */+export function buildPromotionEmail(copy: PromotionCopy, user: EmailUser, claimUrl: string | undefined): EmailMessage {+ const from = promotionSenders()[copy.sender];+ if (!from) throw new PermanentSendError(`The ${copy.sender} sender is not configured.`);+ const rendered = renderPromotion(copy, user, claimUrl);+ return {+ from,+ to: user.email,+ replyTo: emailFrom() || from,+ subject: rendered.subject,+ react: PromotionEmail({+ blocks: rendered.blocks,+ cta: rendered.cta,+ preview: rendered.preview,+ unsubscribeUrl: unsubscribePageUrl(user.id),+ }),+ attachments: [+ {+ content: DONKEY_LOGO_PNG_BASE64,+ contentId: DONKEY_LOGO_CID,+ contentType: "image/png",+ filename: "donkey-cut.png",},+ ],+ headers: {+ "List-Unsubscribe": `<${unsubscribeActionUrl(user.id)}>`,+ "List-Unsubscribe-Post": "List-Unsubscribe=One-Click",},- { idempotencyKey: input.idempotencyKey },- );- if (error) {- throw new Error(`Resend send failed: ${error.name}: ${error.message}`);- }+ };}export type PromotionSegment = { audience: Audience; excludePromotionIds: string[] };@@ -127,10 +110,10 @@ export async function resolvePromotionSegment(where: { marketingUnsubscribedAt: { not: null } },}),segment.excludePromotionIds.length > 0- ? prisma.promotionRecipient.findMany({+ ? prisma.emailSend.findMany({distinct: ["userId"],select: { userId: true },- where: { promotionId: { in: segment.excludePromotionIds }, sentAt: { not: null } },+ where: { promotionId: { in: segment.excludePromotionIds }, state: "sent" },}): Promise.resolve([]),]);@@ -184,6 +167,7 @@ function summarize(): PromotionSummary {return {id: row.id,+ creditOffer: promotionOfferOf(row.creditOffer),name: row.name,subject: row.subject,body: row.body,@@ -205,16 +189,16 @@ function summarize(export async function listPromotions(): Promise<PromotionSummary[]> {const [rows, all, sent, failed] = await Promise.all([prisma.promotion.findMany({ orderBy: { createdAt: "desc" } }),- prisma.promotionRecipient.groupBy({ _count: true, by: ["promotionId"] }),- prisma.promotionRecipient.groupBy({ _count: true, by: ["promotionId"], where: { sentAt: { not: null } } }),- prisma.promotionRecipient.groupBy({+ prisma.emailSend.groupBy({ _count: true, by: ["promotionId"], where: { promotionId: { not: null } } }),+ prisma.emailSend.groupBy({ _count: true, by: ["promotionId"], where: { promotionId: { not: null }, state: "sent" } }),+ prisma.emailSend.groupBy({_count: true,by: ["promotionId"],- where: { attempts: { gte: MAX_SEND_ATTEMPTS }, sentAt: null },+ where: { promotionId: { not: null }, state: { in: ["failed", "skipped"] } },}),]);- const countOf = (groups: { promotionId: string; _count: number }[]) =>- new Map(groups.map((g) => [g.promotionId, g._count]));+ const countOf = (groups: { promotionId: string | null; _count: number }[]) =>+ new Map(groups.flatMap((g) => (g.promotionId ? [[g.promotionId, g._count] as const] : [])));const recipients = countOf(all);const sentBy = countOf(sent);const failedBy = countOf(failed);diff --git a/site/src/lib/marketing/send-outreach.ts b/site/src/lib/marketing/send-outreach.tsindex d891077e..15985645 100644--- a/site/src/lib/marketing/send-outreach.ts+++ b/site/src/lib/marketing/send-outreach.ts@@ -1,45 +1,43 @@-import {- emailFrom,- getResend,- isResendConfigured,- type EmailUser,-} from "@/lib/email/resend";+import { z } from "zod";++import { PermanentSendError } from "@/lib/email/errors";+import { emailFrom, type EmailMessage, type EmailUser } from "@/lib/email/resend";import { isMarketingUnsubscribed, unsubscribePageUrl } from "@/lib/email/unsubscribe";import { outreachReplyAddress } from "@/lib/marketing/replyAddress";-import {- fillOutreachText,- UnknownPlaceholderError,- type OutreachVars,-} from "@/lib/marketing/placeholders";+import { fillOutreachText, type OutreachVars, UnknownPlaceholderError } from "@/lib/marketing/placeholders";+import { prisma } from "@/lib/prisma";-export class OutreachNotSendableError extends Error {- public constructor(message: string) {- super(message);- this.name = "OutreachNotSendableError";- }-}+export const outreachPayloadSchema = z+ .object({+ outreachId: z.string().min(1),+ // Which send this is for the row, so a retried click cannot double-send+ // and a deliberate second note still goes out.+ attempt: z.number().int().min(1),+ actorUserId: z.string().min(1),+ // The final words, from a template or typed in the dialog. Placeholders+ // are filled at send time so the operator edits `{{firstName}}` and the+ // recipient reads their name.+ subject: z.string().min(1),+ body: z.string().min(1),+ vars: z+ .object({ balance: z.string(), email: z.string(), firstName: z.string(), name: z.string(), spent: z.string(), storage: z.string() })+ .strict() satisfies z.ZodType<OutreachVars>,+ // Whether the note carries the opt-out footer. The operator decides per+ // send; the unsubscribed check always runs.+ unsubscribeLink: z.boolean(),+ // Whether replies route through the row's signed alias, which forwards+ // them to the operator's inbox and marks the row replied. Off puts the+ // sending address itself on the wire, so the note reads like any personal+ // thread and the row is filed with the list's "Mark replied" button.+ trackReplies: z.boolean(),+ })+ .strict();-type SendOutreachInput = {- user: EmailUser;- outreachId: string;- // Which send this is for the row, so a retried click cannot double-send and- // a deliberate second note still goes out.- attempt: number;- // The final words, from a template or typed in the dialog. Placeholders are- // filled here so the operator edits `{{firstName}}` and the recipient reads- // their name.- subject: string;- body: string;- vars: OutreachVars;- // Whether the note carries the opt-out footer. The operator decides per- // send; the unsubscribed check above the send always runs.- unsubscribeLink: boolean;- // Whether replies route through the row's signed alias, which forwards them- // to the operator's inbox and marks the row replied. Off puts the sending- // address itself on the wire, so the note reads like any personal thread and- // the row is filed with the list's "Mark replied" button.- trackReplies: boolean;-};+export type OutreachPayload = z.output<typeof outreachPayloadSchema>;++export function outreachIdempotencyKey(outreachId: string, attempt: number): string {+ return `outreach:${outreachId}:${attempt}`;+}// One person writing to one person, so the message goes out as text/plain with// no markup, no template shell, and no bulk-mail headers. A mail provider reads@@ -52,55 +50,43 @@ function outreachText(body: string, unsubscribeUrl: string | null): string {return `${trimmed}\n\n--\nUnsubscribe from product emails: ${unsubscribeUrl}\n`;}-// Sends one outreach note. The reply target is the operator's call per send:-// the row's own signed alias, or the sending address itself.-export async function sendOutreachEmail({- attempt,- body,- outreachId,- subject,- trackReplies,- unsubscribeLink,- user,- vars,-}: SendOutreachInput): Promise<void> {- if (!isResendConfigured()) {- throw new OutreachNotSendableError("RESEND_API_KEY is not configured.");- }+// Builds one outreach note. The reply target is the operator's call per+// send: the row's own signed alias, or the sending address itself.+export async function buildOutreachEmail(payload: OutreachPayload, user: EmailUser): Promise<EmailMessage> {const from = emailFrom();- if (!from) {- throw new OutreachNotSendableError("RESEND_FROM_EMAIL is not configured.");- }- if (await isMarketingUnsubscribed(user.id)) {- throw new OutreachNotSendableError("That account is unsubscribed.");- }+ if (!from) throw new PermanentSendError("RESEND_FROM_EMAIL is not configured.");+ if (await isMarketingUnsubscribed(user.id)) throw new PermanentSendError("That account is unsubscribed.");- let filledSubject: string;- let filledBody: string;+ let subject: string;+ let body: string;try {- filledSubject = fillOutreachText(subject, vars);- filledBody = fillOutreachText(body, vars);+ subject = fillOutreachText(payload.subject, payload.vars);+ body = fillOutreachText(payload.body, payload.vars);} catch (error) {- if (error instanceof UnknownPlaceholderError) {- throw new OutreachNotSendableError(error.message);- }+ if (error instanceof UnknownPlaceholderError) throw new PermanentSendError(error.message);throw error;}+ return {+ from,+ to: user.email,+ replyTo: payload.trackReplies ? outreachReplyAddress(payload.outreachId) : undefined,+ subject,+ text: outreachText(body, payload.unsubscribeLink ? unsubscribePageUrl(user.id) : null),+ };+}- const { error } = await getResend().emails.send(- {- from,- to: user.email,- replyTo: trackReplies ? outreachReplyAddress(outreachId) : undefined,- subject: filledSubject,- text: outreachText(- filledBody,- unsubscribeLink ? unsubscribePageUrl(user.id) : null,- ),+export async function recordOutreachSent(payload: OutreachPayload): Promise<void> {+ const now = new Date();+ const row = await prisma.userOutreach.findUnique({ select: { firstSentAt: true }, where: { id: payload.outreachId } });+ if (!row) return;+ await prisma.userOutreach.update({+ data: {+ actorUserId: payload.actorUserId,+ firstSentAt: row.firstSentAt ?? now,+ lastSentAt: now,+ sentCount: { increment: 1 },+ status: "sent",},- { idempotencyKey: `outreach:${outreachId}:${attempt}` },- );- if (error) {- throw new Error(`Resend send failed: ${error.name}: ${error.message}`);- }+ where: { id: payload.outreachId },+ });}diff --git a/site/src/lib/onboarding/signup-grants.ts b/site/src/lib/onboarding/signup-grants.tsindex f4720401..0b32477d 100644--- a/site/src/lib/onboarding/signup-grants.ts+++ b/site/src/lib/onboarding/signup-grants.ts@@ -5,7 +5,8 @@ import { creditStringToMicros } from "@/lib/credits/amounts";import { grantCredits } from "@/lib/credits/inference";import { creditGrantExpiry } from "@/lib/credits/top-up";import { type EmailUser } from "@/lib/email/resend";-import { sendWelcomeEmail } from "@/lib/email/send-welcome";+import { deliverEmail } from "@/lib/email/outbox";+import { welcomeIdempotencyKey } from "@/lib/email/send-welcome";import { syncResendContact } from "@/lib/email/sync-contact";import { isDeletedAddress } from "@/lib/onboarding/deleted-account";@@ -35,7 +36,7 @@ export async function provisionSignupGrants(user: EmailUser): Promise<void> {: grantSignupAppCredits(user.id, credits, expiresAt),seedStarterProject(user.id),seedFontsFolder(user.id),- sendWelcomeEmail(user, credits),+ deliverEmail({ idempotencyKey: welcomeIdempotencyKey(user.id), kind: "welcome", payload: { credits }, userId: user.id }),syncResendContact(user),]);diff --git a/site/src/queries/promotions.ts b/site/src/queries/promotions.tsindex dde26327..2f0a07f7 100644--- a/site/src/queries/promotions.ts+++ b/site/src/queries/promotions.ts@@ -57,7 +57,7 @@ export function useSendPromotion() {const queryClient = useQueryClient();return useMutation({mutationFn: (id: string) =>- apiFetch<{ jobId: string; recipients: number }>(+ apiFetch<{ recipients: number }>(`/api/su/promotions/${encodeURIComponent(id)}/send`,{ method: "POST" },),@@ -65,6 +65,18 @@ export function useSendPromotion() {});}+export function useCancelPromotion() {+ const queryClient = useQueryClient();+ return useMutation({+ mutationFn: (id: string) =>+ apiFetch<{ promotions: PromotionSummary[] }>(+ `/api/su/promotions/${encodeURIComponent(id)}/cancel`,+ { method: "POST" },+ ),+ onSuccess: () => queryClient.invalidateQueries({ queryKey: promotionsQueryKey }),+ });+}+// Mails the saved copy to the operator.export function useTestPromotion() {return useMutation({