Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.
TypeScript i18nextnext-authnextjspostgresqlprismareact-emailt3-stacktailwindcsstrpcturborepotypescriptzod
🗃️ Purge queued emails 30 days after they finish (#3436)
Luke Vella65eb3773b9dc
diff --git a/apps/web/src/app/api/house-keeping/[...method]/route.ts b/apps/web/src/app/api/house-keeping/[...method]/route.tsindex 1ad3c1b15..5ceca583e 100644--- a/apps/web/src/app/api/house-keeping/[...method]/route.ts+++ b/apps/web/src/app/api/house-keeping/[...method]/route.ts@@ -238,7 +238,7 @@ app.get("/send-queued-emails", async (c) => {const summary = await runQueuedEmailDelivery({});// Runs every minute and most runs find nothing.- if (summary.attempted > 0 || summary.abandoned > 0) {+ if (summary.attempted > 0 || summary.abandoned > 0 || summary.purged > 0) {logger.info({ task: "send-queued-emails", ...summary },"Sent queued emails",diff --git a/apps/web/src/emails/queue.ts b/apps/web/src/emails/queue.tsindex c8c396329..3c18260a4 100644--- a/apps/web/src/emails/queue.ts+++ b/apps/web/src/emails/queue.ts@@ -96,7 +96,11 @@ export function startQueuedEmailScheduler() {running = true;try {const summary = await runQueuedEmailDelivery({});- if (summary.attempted > 0 || summary.abandoned > 0) {+ if (+ summary.attempted > 0 ||+ summary.abandoned > 0 ||+ summary.purged > 0+ ) {logger.info(summary, "Sent queued emails");}} catch (error) {diff --git a/apps/web/src/features/email-queue/constants.ts b/apps/web/src/features/email-queue/constants.tsindex 72c40fa1d..4d65efc42 100644--- a/apps/web/src/features/email-queue/constants.ts+++ b/apps/web/src/features/email-queue/constants.ts@@ -38,6 +38,18 @@ export const QUEUED_EMAIL_CLAIM_TIMEOUT_MS = 10 * 60_000;export const MAX_QUEUED_EMAIL_ATTEMPTS = 3;+/**+ * Emails that reached a final state are deleted after this long. Long+ * enough to answer "did my participants get the email" and to look into+ * abuse, and to cover any per-account email budget window, which must not+ * be longer than this.+ */+export const QUEUED_EMAIL_RETENTION_MS = 30 * 24 * 60 * 60_000;++/** Rows deleted per statement, and statements per run, when purging. */+export const QUEUED_EMAIL_PURGE_CHUNK_SIZE = 1000;+export const QUEUED_EMAIL_PURGE_MAX_CHUNKS = 20;+/*** A run stops starting sends after this long and hands the rest of its* claims back. Together with the rate wait it bounds how late a run candiff --git a/apps/web/src/features/email-queue/data.ts b/apps/web/src/features/email-queue/data.tsindex f7c893c1f..b1fe0163a 100644--- a/apps/web/src/features/email-queue/data.ts+++ b/apps/web/src/features/email-queue/data.ts@@ -14,3 +14,23 @@ export function listQueuedEmails({ ids }: { ids: string[] }) {},});}++/** Emails in a final state last updated before the cutoff, oldest first. */+export async function listPurgeableQueuedEmailIds({+ cutoff,+ limit,+}: {+ cutoff: Date;+ limit: number;+}) {+ const rows = await prisma.queuedEmail.findMany({+ where: {+ status: { in: ["sent", "skipped", "failed"] },+ updatedAt: { lt: cutoff },+ },+ orderBy: { updatedAt: "asc" },+ select: { id: true },+ take: limit,+ });+ return rows.map((row) => row.id);+}diff --git a/apps/web/src/features/email-queue/mutations.ts b/apps/web/src/features/email-queue/mutations.tsindex 3d97f66ac..6cfb5b76f 100644--- a/apps/web/src/features/email-queue/mutations.ts+++ b/apps/web/src/features/email-queue/mutations.ts@@ -11,9 +11,12 @@ import {QUEUED_EMAIL_CLAIM_TIMEOUT_MS,QUEUED_EMAIL_CONCURRENCY,QUEUED_EMAIL_KINDS,+ QUEUED_EMAIL_PURGE_CHUNK_SIZE,+ QUEUED_EMAIL_PURGE_MAX_CHUNKS,+ QUEUED_EMAIL_RETENTION_MS,QUEUED_EMAIL_RUN_BUDGET_MS,} from "./constants";-import { listQueuedEmails } from "./data";+import { listPurgeableQueuedEmailIds, listQueuedEmails } from "./data";import { QueuedEmailHandlers, SendRateLimiter } from "./service";import type { QueuedEmailKind } from "./types";@@ -77,6 +80,39 @@ export const failAbandonedQueuedEmails = Effect.fn(return count;});+/**+ * Deletes emails that reached a final state longer ago than the retention+ * period, a chunk at a time so a backlog never becomes one long statement.+ * Deleting a row twice is harmless, so overlapping runs need no locking.+ * A run stops after a fixed number of chunks; the next run continues.+ * Pending emails are never purged.+ */+export const purgeQueuedEmails = Effect.fn("emailQueue.purgeQueuedEmails")(+ function* ({ now }: { now: Date }) {+ const cutoff = new Date(now.getTime() - QUEUED_EMAIL_RETENTION_MS);+ let purged = 0;+ for (let chunk = 0; chunk < QUEUED_EMAIL_PURGE_MAX_CHUNKS; chunk++) {+ const ids = yield* fromPrisma(() =>+ listPurgeableQueuedEmailIds({+ cutoff,+ limit: QUEUED_EMAIL_PURGE_CHUNK_SIZE,+ }),+ );+ if (ids.length === 0) {+ break;+ }+ const { count } = yield* fromPrisma(() =>+ prisma.queuedEmail.deleteMany({ where: { id: { in: ids } } }),+ );+ purged += count;+ if (ids.length < QUEUED_EMAIL_PURGE_CHUNK_SIZE) {+ break;+ }+ }+ return purged;+ },+);+/*** Claims a batch of pending emails in one statement. SKIP LOCKED lets* overlapping runs claim disjoint batches instead of queueing behind each@@ -253,13 +289,14 @@ export const releaseQueuedEmail = Effect.fn("emailQueue.releaseQueuedEmail")();export type DeliverQueuedEmailsSummary = Record<- QueuedEmailOutcome | "attempted" | "abandoned",+ QueuedEmailOutcome | "attempted" | "abandoned" | "purged",number>;/**- * One run: fail abandoned final attempts, claim a batch, send it a few at a- * time under the shared send rate and record every outcome. The scheduled+ * One run: purge old emails, fail abandoned final attempts, claim a batch,+ * send it a few at a time under the shared send rate and record every+ * outcome. The scheduled* run covers every batch; a run triggered by the action that queued the* emails is limited to that action's batch so its first emails go out at* once. Past the run budget, or when the shared rate has no slot in time,@@ -279,6 +316,21 @@ export const deliverQueuedEmails = Effect.fn("emailQueue.deliverQueuedEmails")(DatabaseError,QueuedEmailHandlers | SendRateLimiter> {+ // Housekeeping: a purge that fails is retried next run, and must not+ // stop this one sending.+ const purged = batchId+ ? 0+ : yield* purgeQueuedEmails({ now }).pipe(+ Effect.catchTag("DatabaseError", (error) =>+ Effect.sync(() => {+ logger.error(+ { error: error.cause },+ "Failed to purge queued emails",+ );+ return 0;+ }),+ ),+ );const abandoned = batchId ? 0 : yield* failAbandonedQueuedEmails({ now });const ids = yield* claimQueuedEmails({ now, limit, batchId });const emails =@@ -307,6 +359,7 @@ export const deliverQueuedEmails = Effect.fn("emailQueue.deliverQueuedEmails")();const summary: DeliverQueuedEmailsSummary = {+ purged,abandoned,attempted: emails.length,sent: 0,diff --git a/apps/web/tests/invite-emails.spec.ts b/apps/web/tests/invite-emails.spec.tsindex 7ffca9c98..80da4bfc0 100644--- a/apps/web/tests/invite-emails.spec.ts+++ b/apps/web/tests/invite-emails.spec.ts@@ -227,6 +227,55 @@ test.describe("Queued invite emails", () => {const row = await findQueued(queued.id);expect(row.status).toBe("failed");});++ test("purges finished emails past the retention period and keeps the rest", async ({+ request,+ }) => {+ const queued = await seed();+ const daysAgo = (days: number) =>+ new Date(Date.now() - days * 24 * 60 * 60_000);+ const base = {+ kind: "scheduled_event_invite" as const,+ batchId: EVENT_ID,+ userId: queued.userId,+ };+ const [oldSent, oldFailed, oldSkipped, recentSent] = await Promise.all([+ prisma.queuedEmail.create({+ data: { ...base, subjectId: "old-sent", status: "sent" },+ }),+ prisma.queuedEmail.create({+ data: { ...base, subjectId: "old-failed", status: "failed" },+ }),+ prisma.queuedEmail.create({+ data: { ...base, subjectId: "old-skipped", status: "skipped" },+ }),+ prisma.queuedEmail.create({+ data: { ...base, subjectId: "recent-sent", status: "sent" },+ }),+ ]);+ // Prisma stamps updatedAt on every write, so age the rows directly. The+ // queued pending email is aged too: pending is never purged.+ await prisma.$executeRaw`+ UPDATE queued_emails SET updated_at = ${daysAgo(31)}+ WHERE id IN (${oldSent.id}, ${oldFailed.id}, ${oldSkipped.id}, ${queued.id})`;+ await prisma.$executeRaw`+ UPDATE queued_emails SET updated_at = ${daysAgo(29)}+ WHERE id = ${recentSent.id}`;++ const summary = await runCron(request);+ expect(summary.purged).toBeGreaterThanOrEqual(3);++ const remaining = await prisma.queuedEmail.findMany({+ where: { batchId: EVENT_ID },+ select: { id: true },+ });+ const ids = remaining.map((row) => row.id);+ expect(ids).not.toContain(oldSent.id);+ expect(ids).not.toContain(oldFailed.id);+ expect(ids).not.toContain(oldSkipped.id);+ expect(ids).toContain(recentSent.id);+ expect(ids).toContain(queued.id);+ });});test.describe("Booking a poll", () => {diff --git a/packages/database/prisma/migrations/20261002130000_index_queued_email_retention/migration.sql b/packages/database/prisma/migrations/20261002130000_index_queued_email_retention/migration.sqlnew file mode 100644index 000000000..7ff861574--- /dev/null+++ b/packages/database/prisma/migrations/20261002130000_index_queued_email_retention/migration.sql@@ -0,0 +1,3 @@+-- CreateIndex+CREATE INDEX "queued_emails_status_updated_at_idx" ON "queued_emails"("status", "updated_at");+diff --git a/packages/database/prisma/models/email-queue.prisma b/packages/database/prisma/models/email-queue.prismaindex f563f5e0b..8f0fcf11c 100644--- a/packages/database/prisma/models/email-queue.prisma+++ b/packages/database/prisma/models/email-queue.prisma@@ -39,6 +39,7 @@ model QueuedEmail {user User @relation(fields: [userId], references: [id], onDelete: Cascade)@@index([status, createdAt])+ @@index([status, updatedAt])@@index([batchId])@@index([userId, status])@@map("queued_emails")