Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Run the normalizedEmail backfill as a local throttled script

Rafi93706fd76512
diff --git a/apps/web/lib/api/customers/backfill-customer-normalized-email.ts b/apps/web/lib/api/customers/backfill-customer-normalized-email.ts
deleted file mode 100644
index bca31b7724..0000000000
--- a/apps/web/lib/api/customers/backfill-customer-normalized-email.ts
+++ /dev/null
@@ -1,94 +0,0 @@
-import { getNormalizedEmail } from "@/lib/email/normalize-email";
-import { prisma } from "@/lib/prisma";
-import { Prisma } from "@prisma/client";
-
-const EMAIL_BATCH_SIZE = 1000;
-
-interface BackfillCustomerNormalizedEmailBatchInput {
- startingAfter?: string;
- dryRun?: boolean;
-}
-
-export async function backfillCustomerNormalizedEmailBatch({
- startingAfter,
- dryRun = false,
-}: BackfillCustomerNormalizedEmailBatchInput) {
- const customers = await prisma.customer.findMany({
- where: {
- ...(startingAfter && {
- id: {
- gt: startingAfter,
- },
- }),
- email: {
- not: null,
- },
- normalizedEmail: null,
- },
- orderBy: {
- id: "asc",
- },
- take: EMAIL_BATCH_SIZE,
- select: {
- id: true,
- email: true,
- },
- });
-
- if (customers.length === 0) {
- return {
- customers,
- updates: [],
- updatedCount: 0,
- lastId: null,
- hasMore: false,
- };
- }
-
- // Skip customers whose email normalizes to null (e.g. empty string)
- // so we match what the write paths store via getNormalizedEmail
- const updates = customers
- .map(({ id, email }) => ({
- id,
- email,
- normalizedEmail: getNormalizedEmail(email),
- }))
- .filter(
- (u): u is { id: string; email: string; normalizedEmail: string } =>
- u.normalizedEmail !== null,
- );
-
- const lastId = customers[customers.length - 1].id;
- const hasMore = customers.length === EMAIL_BATCH_SIZE;
-
- if (dryRun || updates.length === 0) {
- return {
- customers,
- updates,
- updatedCount: 0,
- lastId,
- hasMore,
- };
- }
-
- const updatedCount = await prisma.$executeRaw`
- UPDATE Customer
- SET normalizedEmail = CASE id
- ${Prisma.join(
- updates.map((u) => Prisma.sql`WHEN ${u.id} THEN ${u.normalizedEmail}`),
- " ",
- )}
- END
- WHERE id IN (${Prisma.join(updates.map((u) => u.id))})
- AND normalizedEmail IS NULL
- AND email IS NOT NULL
- `;
-
- return {
- customers,
- updates,
- updatedCount,
- lastId,
- hasMore,
- };
-}
diff --git a/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.ts b/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.ts
deleted file mode 100644
index a0b3d4d57c..0000000000
--- a/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.ts
+++ /dev/null
@@ -1,50 +0,0 @@
-import { backfillCustomerNormalizedEmailBatch } from "@/lib/api/customers/backfill-customer-normalized-email";
-import * as z from "zod/v4";
-import { defineJob } from "../index";
-
-const inputSchema = z.object({
- startingAfter: z.string().optional(),
-});
-
-// TODO: Remove this job (and its entry in jobLoaders, the helper in
-// lib/api/customers/backfill-customer-normalized-email.ts, and the
-// scripts/migrations/backfill-customer-normalized-email.ts script)
-// once the normalizedEmail backfill has finished running in production.
-export const backfillCustomerNormalizedEmailJob = defineJob({
- name: "backfill-customer-normalized-email-job",
- schema: inputSchema,
- defaults: {
- retries: 3,
- flowControl: {
- key: "backfill-customer-normalized-email",
- parallelism: 10,
- },
- },
- async handle(input) {
- const { startingAfter } = input;
-
- const { customers, updates, updatedCount, lastId, hasMore } =
- await backfillCustomerNormalizedEmailBatch({ startingAfter });
-
- if (customers.length === 0) {
- console.log(
- `No more customers to backfill after ${startingAfter ?? "start"}.`,
- );
- return;
- }
-
- if (updates.length === 0) {
- console.log(`No customers to update in this batch. Cursor: ${lastId}`);
- } else {
- console.log(
- `Updated ${updatedCount} of ${customers.length} customers. Cursor: ${lastId}`,
- );
- }
-
- if (hasMore && lastId) {
- await backfillCustomerNormalizedEmailJob.dispatch({
- startingAfter: lastId,
- });
- }
- },
-});
diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.ts
index 6ca6af6246..eb14881eb0 100644
--- a/apps/web/lib/jobs/registry.ts
+++ b/apps/web/lib/jobs/registry.ts
@@ -138,11 +138,6 @@ const jobLoaders = {
import("./handlers/program-application-reminder-job").then(
(m) => m.programApplicationReminderJob,
),
-
- "backfill-customer-normalized-email-job": () =>
- import("./handlers/backfill-customer-normalized-email-job").then(
- (m) => m.backfillCustomerNormalizedEmailJob,
- ),
} as const satisfies Record<string, () => Promise<JobDefinition>>;
const jobCache = new Map<string, JobDefinition>();
diff --git a/apps/web/scripts/migrations/backfill-customer-normalized-email.ts b/apps/web/scripts/migrations/backfill-customer-normalized-email.ts
index 5580d6bdd2..f80618698a 100644
--- a/apps/web/scripts/migrations/backfill-customer-normalized-email.ts
+++ b/apps/web/scripts/migrations/backfill-customer-normalized-email.ts
@@ -1,56 +1,121 @@
-import { backfillCustomerNormalizedEmailBatch } from "@/lib/api/customers/backfill-customer-normalized-email";
-import { backfillCustomerNormalizedEmailJob } from "@/lib/jobs/handlers/backfill-customer-normalized-email-job";
import "dotenv-flow/config";
+import { getNormalizedEmail } from "@/lib/email/normalize-email";
+import { prisma } from "@/lib/prisma";
+import { sleep } from "@dub/utils";
+import { Prisma } from "@prisma/client";
+
+// TODO: Remove this script once the normalizedEmail backfill has finished
+// running in production.
+
const DRY_RUN = true;
+const BATCH_SIZE = 1000;
+const THROTTLE_MS = 500;
+const LAST_CURSOR_ID: string | null = null; // Paste the last printed cursor id here to resume after a crash.
const SAMPLE_SIZE = 20;
async function main() {
- if (!DRY_RUN) {
- await backfillCustomerNormalizedEmailJob.dispatch({});
- console.log("Dispatched backfill-customer-normalized-email-job");
- return;
+ console.log(
+ `DRY_RUN=${DRY_RUN} BATCH_SIZE=${BATCH_SIZE} THROTTLE_MS=${THROTTLE_MS}`,
+ );
+
+ let startingAfter = LAST_CURSOR_ID ?? undefined;
+ if (startingAfter) {
+ console.log(`Resuming from ${startingAfter}`);
}
- let startingAfter: string | undefined;
let totalScanned = 0;
- let totalWouldUpdate = 0;
+ let totalUpdated = 0;
let totalSkipped = 0;
- let sample: { id: string; email: string; normalizedEmail: string }[] = [];
+ const sample: { id: string; email: string; normalizedEmail: string }[] = [];
while (true) {
- const { customers, updates, lastId, hasMore } =
- await backfillCustomerNormalizedEmailBatch({
- startingAfter,
- dryRun: true,
- });
+ const customers = await prisma.customer.findMany({
+ where: {
+ email: {
+ not: null,
+ },
+ normalizedEmail: null,
+ ...(startingAfter && {
+ id: {
+ gt: startingAfter,
+ },
+ }),
+ },
+ select: {
+ id: true,
+ email: true,
+ },
+ take: BATCH_SIZE,
+ orderBy: {
+ id: "asc",
+ },
+ });
if (customers.length === 0) {
break;
}
totalScanned += customers.length;
- totalWouldUpdate += updates.length;
+
+ // Skip emails that normalize to null (e.g. empty strings), to match what
+ // the write paths store via getNormalizedEmail
+ const updates = customers
+ .map(({ id, email }) => ({
+ id,
+ email,
+ normalizedEmail: getNormalizedEmail(email),
+ }))
+ .filter(
+ (u): u is { id: string; email: string; normalizedEmail: string } =>
+ u.normalizedEmail !== null,
+ );
+
totalSkipped += customers.length - updates.length;
if (sample.length < SAMPLE_SIZE) {
- sample = sample.concat(updates.slice(0, SAMPLE_SIZE - sample.length));
+ sample.push(...updates.slice(0, SAMPLE_SIZE - sample.length));
+ }
+
+ let batchUpdated = 0;
+
+ // The IS NULL guards skip rows that a write path or a redaction changed
+ // after this batch was read
+ if (!DRY_RUN && updates.length > 0) {
+ batchUpdated = await prisma.$executeRaw`
+ UPDATE Customer
+ SET normalizedEmail = CASE id
+ ${Prisma.join(
+ updates.map(
+ (u) => Prisma.sql`WHEN ${u.id} THEN ${u.normalizedEmail}`,
+ ),
+ " ",
+ )}
+ END
+ WHERE id IN (${Prisma.join(updates.map((u) => u.id))})
+ AND normalizedEmail IS NULL
+ AND email IS NOT NULL
+ `;
+
+ totalUpdated += batchUpdated;
}
+ startingAfter = customers[customers.length - 1].id;
+
console.log(
- `Would update ${updates.length} of ${customers.length} customers. Cursor: ${lastId}`,
+ `${DRY_RUN ? "Would update" : "Updated"} ${DRY_RUN ? updates.length : batchUpdated} of ${customers.length} customers. Cursor: ${startingAfter}`,
);
- if (!hasMore || !lastId) {
+ if (customers.length < BATCH_SIZE) {
break;
}
- startingAfter = lastId;
+ await sleep(THROTTLE_MS);
}
console.table(sample);
console.log(
- `DRY_RUN=${DRY_RUN} scanned=${totalScanned} would-update=${totalWouldUpdate} skipped=${totalSkipped}`,
+ `DRY_RUN=${DRY_RUN} scanned=${totalScanned} updated=${totalUpdated} skipped=${totalSkipped}`,
);
}