The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Run the normalizedEmail backfill as a local throttled script
Rafi93706fd76512
diff --git a/apps/web/lib/api/customers/backfill-customer-normalized-email.ts b/apps/web/lib/api/customers/backfill-customer-normalized-email.tsdeleted file mode 100644index bca31b7724..0000000000--- a/apps/web/lib/api/customers/backfill-customer-normalized-email.ts+++ /dev/null@@ -1,94 +0,0 @@-import { getNormalizedEmail } from "@/lib/email/normalize-email";-import { prisma } from "@/lib/prisma";-import { Prisma } from "@prisma/client";--const EMAIL_BATCH_SIZE = 1000;--interface BackfillCustomerNormalizedEmailBatchInput {- startingAfter?: string;- dryRun?: boolean;-}--export async function backfillCustomerNormalizedEmailBatch({- startingAfter,- dryRun = false,-}: BackfillCustomerNormalizedEmailBatchInput) {- const customers = await prisma.customer.findMany({- where: {- ...(startingAfter && {- id: {- gt: startingAfter,- },- }),- email: {- not: null,- },- normalizedEmail: null,- },- orderBy: {- id: "asc",- },- take: EMAIL_BATCH_SIZE,- select: {- id: true,- email: true,- },- });-- if (customers.length === 0) {- return {- customers,- updates: [],- updatedCount: 0,- lastId: null,- hasMore: false,- };- }-- // Skip customers whose email normalizes to null (e.g. empty string)- // so we match what the write paths store via getNormalizedEmail- const updates = customers- .map(({ id, email }) => ({- id,- email,- normalizedEmail: getNormalizedEmail(email),- }))- .filter(- (u): u is { id: string; email: string; normalizedEmail: string } =>- u.normalizedEmail !== null,- );-- const lastId = customers[customers.length - 1].id;- const hasMore = customers.length === EMAIL_BATCH_SIZE;-- if (dryRun || updates.length === 0) {- return {- customers,- updates,- updatedCount: 0,- lastId,- hasMore,- };- }-- const updatedCount = await prisma.$executeRaw`- UPDATE Customer- SET normalizedEmail = CASE id- ${Prisma.join(- updates.map((u) => Prisma.sql`WHEN ${u.id} THEN ${u.normalizedEmail}`),- " ",- )}- END- WHERE id IN (${Prisma.join(updates.map((u) => u.id))})- AND normalizedEmail IS NULL- AND email IS NOT NULL- `;-- return {- customers,- updates,- updatedCount,- lastId,- hasMore,- };-}diff --git a/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.ts b/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.tsdeleted file mode 100644index a0b3d4d57c..0000000000--- a/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.ts+++ /dev/null@@ -1,50 +0,0 @@-import { backfillCustomerNormalizedEmailBatch } from "@/lib/api/customers/backfill-customer-normalized-email";-import * as z from "zod/v4";-import { defineJob } from "../index";--const inputSchema = z.object({- startingAfter: z.string().optional(),-});--// TODO: Remove this job (and its entry in jobLoaders, the helper in-// lib/api/customers/backfill-customer-normalized-email.ts, and the-// scripts/migrations/backfill-customer-normalized-email.ts script)-// once the normalizedEmail backfill has finished running in production.-export const backfillCustomerNormalizedEmailJob = defineJob({- name: "backfill-customer-normalized-email-job",- schema: inputSchema,- defaults: {- retries: 3,- flowControl: {- key: "backfill-customer-normalized-email",- parallelism: 10,- },- },- async handle(input) {- const { startingAfter } = input;-- const { customers, updates, updatedCount, lastId, hasMore } =- await backfillCustomerNormalizedEmailBatch({ startingAfter });-- if (customers.length === 0) {- console.log(- `No more customers to backfill after ${startingAfter ?? "start"}.`,- );- return;- }-- if (updates.length === 0) {- console.log(`No customers to update in this batch. Cursor: ${lastId}`);- } else {- console.log(- `Updated ${updatedCount} of ${customers.length} customers. Cursor: ${lastId}`,- );- }-- if (hasMore && lastId) {- await backfillCustomerNormalizedEmailJob.dispatch({- startingAfter: lastId,- });- }- },-});diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.tsindex 6ca6af6246..eb14881eb0 100644--- a/apps/web/lib/jobs/registry.ts+++ b/apps/web/lib/jobs/registry.ts@@ -138,11 +138,6 @@ const jobLoaders = {import("./handlers/program-application-reminder-job").then((m) => m.programApplicationReminderJob,),-- "backfill-customer-normalized-email-job": () =>- import("./handlers/backfill-customer-normalized-email-job").then(- (m) => m.backfillCustomerNormalizedEmailJob,- ),} as const satisfies Record<string, () => Promise<JobDefinition>>;const jobCache = new Map<string, JobDefinition>();diff --git a/apps/web/scripts/migrations/backfill-customer-normalized-email.ts b/apps/web/scripts/migrations/backfill-customer-normalized-email.tsindex 5580d6bdd2..f80618698a 100644--- a/apps/web/scripts/migrations/backfill-customer-normalized-email.ts+++ b/apps/web/scripts/migrations/backfill-customer-normalized-email.ts@@ -1,56 +1,121 @@-import { backfillCustomerNormalizedEmailBatch } from "@/lib/api/customers/backfill-customer-normalized-email";-import { backfillCustomerNormalizedEmailJob } from "@/lib/jobs/handlers/backfill-customer-normalized-email-job";import "dotenv-flow/config";+import { getNormalizedEmail } from "@/lib/email/normalize-email";+import { prisma } from "@/lib/prisma";+import { sleep } from "@dub/utils";+import { Prisma } from "@prisma/client";++// TODO: Remove this script once the normalizedEmail backfill has finished+// running in production.+const DRY_RUN = true;+const BATCH_SIZE = 1000;+const THROTTLE_MS = 500;+const LAST_CURSOR_ID: string | null = null; // Paste the last printed cursor id here to resume after a crash.const SAMPLE_SIZE = 20;async function main() {- if (!DRY_RUN) {- await backfillCustomerNormalizedEmailJob.dispatch({});- console.log("Dispatched backfill-customer-normalized-email-job");- return;+ console.log(+ `DRY_RUN=${DRY_RUN} BATCH_SIZE=${BATCH_SIZE} THROTTLE_MS=${THROTTLE_MS}`,+ );++ let startingAfter = LAST_CURSOR_ID ?? undefined;+ if (startingAfter) {+ console.log(`Resuming from ${startingAfter}`);}- let startingAfter: string | undefined;let totalScanned = 0;- let totalWouldUpdate = 0;+ let totalUpdated = 0;let totalSkipped = 0;- let sample: { id: string; email: string; normalizedEmail: string }[] = [];+ const sample: { id: string; email: string; normalizedEmail: string }[] = [];while (true) {- const { customers, updates, lastId, hasMore } =- await backfillCustomerNormalizedEmailBatch({- startingAfter,- dryRun: true,- });+ const customers = await prisma.customer.findMany({+ where: {+ email: {+ not: null,+ },+ normalizedEmail: null,+ ...(startingAfter && {+ id: {+ gt: startingAfter,+ },+ }),+ },+ select: {+ id: true,+ email: true,+ },+ take: BATCH_SIZE,+ orderBy: {+ id: "asc",+ },+ });if (customers.length === 0) {break;}totalScanned += customers.length;- totalWouldUpdate += updates.length;++ // Skip emails that normalize to null (e.g. empty strings), to match what+ // the write paths store via getNormalizedEmail+ const updates = customers+ .map(({ id, email }) => ({+ id,+ email,+ normalizedEmail: getNormalizedEmail(email),+ }))+ .filter(+ (u): u is { id: string; email: string; normalizedEmail: string } =>+ u.normalizedEmail !== null,+ );+totalSkipped += customers.length - updates.length;if (sample.length < SAMPLE_SIZE) {- sample = sample.concat(updates.slice(0, SAMPLE_SIZE - sample.length));+ sample.push(...updates.slice(0, SAMPLE_SIZE - sample.length));+ }++ let batchUpdated = 0;++ // The IS NULL guards skip rows that a write path or a redaction changed+ // after this batch was read+ if (!DRY_RUN && updates.length > 0) {+ batchUpdated = await prisma.$executeRaw`+ UPDATE Customer+ SET normalizedEmail = CASE id+ ${Prisma.join(+ updates.map(+ (u) => Prisma.sql`WHEN ${u.id} THEN ${u.normalizedEmail}`,+ ),+ " ",+ )}+ END+ WHERE id IN (${Prisma.join(updates.map((u) => u.id))})+ AND normalizedEmail IS NULL+ AND email IS NOT NULL+ `;++ totalUpdated += batchUpdated;}+ startingAfter = customers[customers.length - 1].id;+console.log(- `Would update ${updates.length} of ${customers.length} customers. Cursor: ${lastId}`,+ `${DRY_RUN ? "Would update" : "Updated"} ${DRY_RUN ? updates.length : batchUpdated} of ${customers.length} customers. Cursor: ${startingAfter}`,);- if (!hasMore || !lastId) {+ if (customers.length < BATCH_SIZE) {break;}- startingAfter = lastId;+ await sleep(THROTTLE_MS);}console.table(sample);console.log(- `DRY_RUN=${DRY_RUN} scanned=${totalScanned} would-update=${totalWouldUpdate} skipped=${totalSkipped}`,+ `DRY_RUN=${DRY_RUN} scanned=${totalScanned} updated=${totalUpdated} skipped=${totalSkipped}`,);}