Skip to content

Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.

🗃️ Purge queued emails 30 days after they finish (#3436)

Luke Vella65eb3773b9dc
diff --git a/apps/web/src/app/api/house-keeping/[...method]/route.ts b/apps/web/src/app/api/house-keeping/[...method]/route.ts
index 1ad3c1b15..5ceca583e 100644
--- a/apps/web/src/app/api/house-keeping/[...method]/route.ts
+++ b/apps/web/src/app/api/house-keeping/[...method]/route.ts
@@ -238,7 +238,7 @@ app.get("/send-queued-emails", async (c) => {
const summary = await runQueuedEmailDelivery({});
// Runs every minute and most runs find nothing.
- if (summary.attempted > 0 || summary.abandoned > 0) {
+ if (summary.attempted > 0 || summary.abandoned > 0 || summary.purged > 0) {
logger.info(
{ task: "send-queued-emails", ...summary },
"Sent queued emails",
diff --git a/apps/web/src/emails/queue.ts b/apps/web/src/emails/queue.ts
index c8c396329..3c18260a4 100644
--- a/apps/web/src/emails/queue.ts
+++ b/apps/web/src/emails/queue.ts
@@ -96,7 +96,11 @@ export function startQueuedEmailScheduler() {
running = true;
try {
const summary = await runQueuedEmailDelivery({});
- if (summary.attempted > 0 || summary.abandoned > 0) {
+ if (
+ summary.attempted > 0 ||
+ summary.abandoned > 0 ||
+ summary.purged > 0
+ ) {
logger.info(summary, "Sent queued emails");
}
} catch (error) {
diff --git a/apps/web/src/features/email-queue/constants.ts b/apps/web/src/features/email-queue/constants.ts
index 72c40fa1d..4d65efc42 100644
--- a/apps/web/src/features/email-queue/constants.ts
+++ b/apps/web/src/features/email-queue/constants.ts
@@ -38,6 +38,18 @@ export const QUEUED_EMAIL_CLAIM_TIMEOUT_MS = 10 * 60_000;
export const MAX_QUEUED_EMAIL_ATTEMPTS = 3;
+/**
+ * Emails that reached a final state are deleted after this long. Long
+ * enough to answer "did my participants get the email" and to look into
+ * abuse, and to cover any per-account email budget window, which must not
+ * be longer than this.
+ */
+export const QUEUED_EMAIL_RETENTION_MS = 30 * 24 * 60 * 60_000;
+
+/** Rows deleted per statement, and statements per run, when purging. */
+export const QUEUED_EMAIL_PURGE_CHUNK_SIZE = 1000;
+export const QUEUED_EMAIL_PURGE_MAX_CHUNKS = 20;
+
/**
* A run stops starting sends after this long and hands the rest of its
* claims back. Together with the rate wait it bounds how late a run can
diff --git a/apps/web/src/features/email-queue/data.ts b/apps/web/src/features/email-queue/data.ts
index f7c893c1f..b1fe0163a 100644
--- a/apps/web/src/features/email-queue/data.ts
+++ b/apps/web/src/features/email-queue/data.ts
@@ -14,3 +14,23 @@ export function listQueuedEmails({ ids }: { ids: string[] }) {
},
});
}
+
+/** Emails in a final state last updated before the cutoff, oldest first. */
+export async function listPurgeableQueuedEmailIds({
+ cutoff,
+ limit,
+}: {
+ cutoff: Date;
+ limit: number;
+}) {
+ const rows = await prisma.queuedEmail.findMany({
+ where: {
+ status: { in: ["sent", "skipped", "failed"] },
+ updatedAt: { lt: cutoff },
+ },
+ orderBy: { updatedAt: "asc" },
+ select: { id: true },
+ take: limit,
+ });
+ return rows.map((row) => row.id);
+}
diff --git a/apps/web/src/features/email-queue/mutations.ts b/apps/web/src/features/email-queue/mutations.ts
index 3d97f66ac..6cfb5b76f 100644
--- a/apps/web/src/features/email-queue/mutations.ts
+++ b/apps/web/src/features/email-queue/mutations.ts
@@ -11,9 +11,12 @@ import {
QUEUED_EMAIL_CLAIM_TIMEOUT_MS,
QUEUED_EMAIL_CONCURRENCY,
QUEUED_EMAIL_KINDS,
+ QUEUED_EMAIL_PURGE_CHUNK_SIZE,
+ QUEUED_EMAIL_PURGE_MAX_CHUNKS,
+ QUEUED_EMAIL_RETENTION_MS,
QUEUED_EMAIL_RUN_BUDGET_MS,
} from "./constants";
-import { listQueuedEmails } from "./data";
+import { listPurgeableQueuedEmailIds, listQueuedEmails } from "./data";
import { QueuedEmailHandlers, SendRateLimiter } from "./service";
import type { QueuedEmailKind } from "./types";
@@ -77,6 +80,39 @@ export const failAbandonedQueuedEmails = Effect.fn(
return count;
});
+/**
+ * Deletes emails that reached a final state longer ago than the retention
+ * period, a chunk at a time so a backlog never becomes one long statement.
+ * Deleting a row twice is harmless, so overlapping runs need no locking.
+ * A run stops after a fixed number of chunks; the next run continues.
+ * Pending emails are never purged.
+ */
+export const purgeQueuedEmails = Effect.fn("emailQueue.purgeQueuedEmails")(
+ function* ({ now }: { now: Date }) {
+ const cutoff = new Date(now.getTime() - QUEUED_EMAIL_RETENTION_MS);
+ let purged = 0;
+ for (let chunk = 0; chunk < QUEUED_EMAIL_PURGE_MAX_CHUNKS; chunk++) {
+ const ids = yield* fromPrisma(() =>
+ listPurgeableQueuedEmailIds({
+ cutoff,
+ limit: QUEUED_EMAIL_PURGE_CHUNK_SIZE,
+ }),
+ );
+ if (ids.length === 0) {
+ break;
+ }
+ const { count } = yield* fromPrisma(() =>
+ prisma.queuedEmail.deleteMany({ where: { id: { in: ids } } }),
+ );
+ purged += count;
+ if (ids.length < QUEUED_EMAIL_PURGE_CHUNK_SIZE) {
+ break;
+ }
+ }
+ return purged;
+ },
+);
+
/**
* Claims a batch of pending emails in one statement. SKIP LOCKED lets
* overlapping runs claim disjoint batches instead of queueing behind each
@@ -253,13 +289,14 @@ export const releaseQueuedEmail = Effect.fn("emailQueue.releaseQueuedEmail")(
);
export type DeliverQueuedEmailsSummary = Record<
- QueuedEmailOutcome | "attempted" | "abandoned",
+ QueuedEmailOutcome | "attempted" | "abandoned" | "purged",
number
>;
/**
- * One run: fail abandoned final attempts, claim a batch, send it a few at a
- * time under the shared send rate and record every outcome. The scheduled
+ * One run: purge old emails, fail abandoned final attempts, claim a batch,
+ * send it a few at a time under the shared send rate and record every
+ * outcome. The scheduled
* run covers every batch; a run triggered by the action that queued the
* emails is limited to that action's batch so its first emails go out at
* once. Past the run budget, or when the shared rate has no slot in time,
@@ -279,6 +316,21 @@ export const deliverQueuedEmails = Effect.fn("emailQueue.deliverQueuedEmails")(
DatabaseError,
QueuedEmailHandlers | SendRateLimiter
> {
+ // Housekeeping: a purge that fails is retried next run, and must not
+ // stop this one sending.
+ const purged = batchId
+ ? 0
+ : yield* purgeQueuedEmails({ now }).pipe(
+ Effect.catchTag("DatabaseError", (error) =>
+ Effect.sync(() => {
+ logger.error(
+ { error: error.cause },
+ "Failed to purge queued emails",
+ );
+ return 0;
+ }),
+ ),
+ );
const abandoned = batchId ? 0 : yield* failAbandonedQueuedEmails({ now });
const ids = yield* claimQueuedEmails({ now, limit, batchId });
const emails =
@@ -307,6 +359,7 @@ export const deliverQueuedEmails = Effect.fn("emailQueue.deliverQueuedEmails")(
);
const summary: DeliverQueuedEmailsSummary = {
+ purged,
abandoned,
attempted: emails.length,
sent: 0,
diff --git a/apps/web/tests/invite-emails.spec.ts b/apps/web/tests/invite-emails.spec.ts
index 7ffca9c98..80da4bfc0 100644
--- a/apps/web/tests/invite-emails.spec.ts
+++ b/apps/web/tests/invite-emails.spec.ts
@@ -227,6 +227,55 @@ test.describe("Queued invite emails", () => {
const row = await findQueued(queued.id);
expect(row.status).toBe("failed");
});
+
+ test("purges finished emails past the retention period and keeps the rest", async ({
+ request,
+ }) => {
+ const queued = await seed();
+ const daysAgo = (days: number) =>
+ new Date(Date.now() - days * 24 * 60 * 60_000);
+ const base = {
+ kind: "scheduled_event_invite" as const,
+ batchId: EVENT_ID,
+ userId: queued.userId,
+ };
+ const [oldSent, oldFailed, oldSkipped, recentSent] = await Promise.all([
+ prisma.queuedEmail.create({
+ data: { ...base, subjectId: "old-sent", status: "sent" },
+ }),
+ prisma.queuedEmail.create({
+ data: { ...base, subjectId: "old-failed", status: "failed" },
+ }),
+ prisma.queuedEmail.create({
+ data: { ...base, subjectId: "old-skipped", status: "skipped" },
+ }),
+ prisma.queuedEmail.create({
+ data: { ...base, subjectId: "recent-sent", status: "sent" },
+ }),
+ ]);
+ // Prisma stamps updatedAt on every write, so age the rows directly. The
+ // queued pending email is aged too: pending is never purged.
+ await prisma.$executeRaw`
+ UPDATE queued_emails SET updated_at = ${daysAgo(31)}
+ WHERE id IN (${oldSent.id}, ${oldFailed.id}, ${oldSkipped.id}, ${queued.id})`;
+ await prisma.$executeRaw`
+ UPDATE queued_emails SET updated_at = ${daysAgo(29)}
+ WHERE id = ${recentSent.id}`;
+
+ const summary = await runCron(request);
+ expect(summary.purged).toBeGreaterThanOrEqual(3);
+
+ const remaining = await prisma.queuedEmail.findMany({
+ where: { batchId: EVENT_ID },
+ select: { id: true },
+ });
+ const ids = remaining.map((row) => row.id);
+ expect(ids).not.toContain(oldSent.id);
+ expect(ids).not.toContain(oldFailed.id);
+ expect(ids).not.toContain(oldSkipped.id);
+ expect(ids).toContain(recentSent.id);
+ expect(ids).toContain(queued.id);
+ });
});
test.describe("Booking a poll", () => {
diff --git a/packages/database/prisma/migrations/20261002130000_index_queued_email_retention/migration.sql b/packages/database/prisma/migrations/20261002130000_index_queued_email_retention/migration.sql
new file mode 100644
index 000000000..7ff861574
--- /dev/null
+++ b/packages/database/prisma/migrations/20261002130000_index_queued_email_retention/migration.sql
@@ -0,0 +1,3 @@
+-- CreateIndex
+CREATE INDEX "queued_emails_status_updated_at_idx" ON "queued_emails"("status", "updated_at");
+
diff --git a/packages/database/prisma/models/email-queue.prisma b/packages/database/prisma/models/email-queue.prisma
index f563f5e0b..8f0fcf11c 100644
--- a/packages/database/prisma/models/email-queue.prisma
+++ b/packages/database/prisma/models/email-queue.prisma
@@ -39,6 +39,7 @@ model QueuedEmail {
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@index([status, createdAt])
+ @@index([status, updatedAt])
@@index([batchId])
@@index([userId, status])
@@map("queued_emails")