The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Add normalizedEmail backfill job and similar-email fraud e2e test
Kiran Ke4f538de6ebb
diff --git a/apps/web/lib/api/customers/backfill-customer-normalized-email.ts b/apps/web/lib/api/customers/backfill-customer-normalized-email.tsnew file mode 100644index 0000000000..c1af40ab76--- /dev/null+++ b/apps/web/lib/api/customers/backfill-customer-normalized-email.ts@@ -0,0 +1,93 @@+import { getNormalizedEmail } from "@/lib/email/normalize-email";+import { prisma } from "@/lib/prisma";+import { Prisma } from "@prisma/client";++const EMAIL_BATCH_SIZE = 1000;++interface BackfillCustomerNormalizedEmailBatchInput {+ startingAfter?: string;+ dryRun?: boolean;+}++export async function backfillCustomerNormalizedEmailBatch({+ startingAfter,+ dryRun = false,+}: BackfillCustomerNormalizedEmailBatchInput) {+ const customers = await prisma.customer.findMany({+ where: {+ ...(startingAfter && {+ id: {+ gt: startingAfter,+ },+ }),+ email: {+ not: null,+ },+ normalizedEmail: null,+ },+ orderBy: {+ id: "asc",+ },+ take: EMAIL_BATCH_SIZE,+ select: {+ id: true,+ email: true,+ },+ });++ if (customers.length === 0) {+ return {+ customers,+ updates: [],+ updatedCount: 0,+ lastId: null,+ hasMore: false,+ };+ }++ // Skip customers whose email normalizes to null (e.g. empty string)+ // so we match what the write paths store via getNormalizedEmail+ const updates = customers+ .map(({ id, email }) => ({+ id,+ email,+ normalizedEmail: getNormalizedEmail(email),+ }))+ .filter(+ (u): u is { id: string; email: string; normalizedEmail: string } =>+ u.normalizedEmail !== null,+ );++ const lastId = customers[customers.length - 1].id;+ const hasMore = customers.length === EMAIL_BATCH_SIZE;++ if (dryRun || updates.length === 0) {+ return {+ customers,+ updates,+ updatedCount: 0,+ lastId,+ hasMore,+ };+ }++ const updatedCount = await prisma.$executeRaw`+ UPDATE Customer+ SET normalizedEmail = CASE id+ ${Prisma.join(+ updates.map((u) => Prisma.sql`WHEN ${u.id} THEN ${u.normalizedEmail}`),+ " ",+ )}+ END+ WHERE id IN (${Prisma.join(updates.map((u) => u.id))})+ AND normalizedEmail IS NULL+ `;++ return {+ customers,+ updates,+ updatedCount,+ lastId,+ hasMore,+ };+}diff --git a/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.ts b/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.tsnew file mode 100644index 0000000000..a0b3d4d57c--- /dev/null+++ b/apps/web/lib/jobs/handlers/backfill-customer-normalized-email-job.ts@@ -0,0 +1,50 @@+import { backfillCustomerNormalizedEmailBatch } from "@/lib/api/customers/backfill-customer-normalized-email";+import * as z from "zod/v4";+import { defineJob } from "../index";++const inputSchema = z.object({+ startingAfter: z.string().optional(),+});++// TODO: Remove this job (and its entry in jobLoaders, the helper in+// lib/api/customers/backfill-customer-normalized-email.ts, and the+// scripts/migrations/backfill-customer-normalized-email.ts script)+// once the normalizedEmail backfill has finished running in production.+export const backfillCustomerNormalizedEmailJob = defineJob({+ name: "backfill-customer-normalized-email-job",+ schema: inputSchema,+ defaults: {+ retries: 3,+ flowControl: {+ key: "backfill-customer-normalized-email",+ parallelism: 10,+ },+ },+ async handle(input) {+ const { startingAfter } = input;++ const { customers, updates, updatedCount, lastId, hasMore } =+ await backfillCustomerNormalizedEmailBatch({ startingAfter });++ if (customers.length === 0) {+ console.log(+ `No more customers to backfill after ${startingAfter ?? "start"}.`,+ );+ return;+ }++ if (updates.length === 0) {+ console.log(`No customers to update in this batch. Cursor: ${lastId}`);+ } else {+ console.log(+ `Updated ${updatedCount} of ${customers.length} customers. Cursor: ${lastId}`,+ );+ }++ if (hasMore && lastId) {+ await backfillCustomerNormalizedEmailJob.dispatch({+ startingAfter: lastId,+ });+ }+ },+});diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.tsindex eb14881eb0..6ca6af6246 100644--- a/apps/web/lib/jobs/registry.ts+++ b/apps/web/lib/jobs/registry.ts@@ -138,6 +138,11 @@ const jobLoaders = {import("./handlers/program-application-reminder-job").then((m) => m.programApplicationReminderJob,),++ "backfill-customer-normalized-email-job": () =>+ import("./handlers/backfill-customer-normalized-email-job").then(+ (m) => m.backfillCustomerNormalizedEmailJob,+ ),} as const satisfies Record<string, () => Promise<JobDefinition>>;const jobCache = new Map<string, JobDefinition>();diff --git a/apps/web/scripts/migrations/backfill-customer-normalized-email.ts b/apps/web/scripts/migrations/backfill-customer-normalized-email.tsnew file mode 100644index 0000000000..5580d6bdd2--- /dev/null+++ b/apps/web/scripts/migrations/backfill-customer-normalized-email.ts@@ -0,0 +1,57 @@+import { backfillCustomerNormalizedEmailBatch } from "@/lib/api/customers/backfill-customer-normalized-email";+import { backfillCustomerNormalizedEmailJob } from "@/lib/jobs/handlers/backfill-customer-normalized-email-job";+import "dotenv-flow/config";++const DRY_RUN = true;+const SAMPLE_SIZE = 20;++async function main() {+ if (!DRY_RUN) {+ await backfillCustomerNormalizedEmailJob.dispatch({});+ console.log("Dispatched backfill-customer-normalized-email-job");+ return;+ }++ let startingAfter: string | undefined;+ let totalScanned = 0;+ let totalWouldUpdate = 0;+ let totalSkipped = 0;+ let sample: { id: string; email: string; normalizedEmail: string }[] = [];++ while (true) {+ const { customers, updates, lastId, hasMore } =+ await backfillCustomerNormalizedEmailBatch({+ startingAfter,+ dryRun: true,+ });++ if (customers.length === 0) {+ break;+ }++ totalScanned += customers.length;+ totalWouldUpdate += updates.length;+ totalSkipped += customers.length - updates.length;++ if (sample.length < SAMPLE_SIZE) {+ sample = sample.concat(updates.slice(0, SAMPLE_SIZE - sample.length));+ }++ console.log(+ `Would update ${updates.length} of ${customers.length} customers. Cursor: ${lastId}`,+ );++ if (!hasMore || !lastId) {+ break;+ }++ startingAfter = lastId;+ }++ console.table(sample);+ console.log(+ `DRY_RUN=${DRY_RUN} scanned=${totalScanned} would-update=${totalWouldUpdate} skipped=${totalSkipped}`,+ );+}++main();diff --git a/apps/web/tests/fraud/index.test.ts b/apps/web/tests/fraud/index.test.tsindex dd9b5b6e87..f3a45ce731 100644--- a/apps/web/tests/fraud/index.test.ts+++ b/apps/web/tests/fraud/index.test.ts@@ -1,16 +1,24 @@import { extractEmailDomain } from "@/lib/email/extract-email-domain";import { TrackLeadResponse } from "@/lib/types";import { CustomerEmailMatchType } from "@/lib/zod/schemas/fraud";-import { randomCustomer } from "tests/utils/helpers";+import { randomCustomer, randomId } from "tests/utils/helpers";import {E2E_FRAUD_PARTNER,E2E_FRAUD_REFERRAL_SOURCE_BANNED_DOMAIN,E2E_TRACK_CLICK_HEADERS,} from "tests/utils/resource";import { verifyFraudEvent } from "tests/utils/verify-fraud-event";-import { describe, test } from "vitest";+import { describe, expect, test } from "vitest";import { IntegrationHarness } from "../utils/integration";+const randomEmailLocalPart = () =>+ `e2e${randomId(12)+ .replace(/[^a-zA-Z0-9]/g, "")+ .toLowerCase()}`;++const randomUniqueEmail = (domain: string) =>+ `${randomEmailLocalPart()}@${domain}`;+describe.concurrent("/fraud/**", async () => {const h = new IntegrationHarness();const { http } = await h.init();@@ -72,9 +80,10 @@ describe.concurrent("/fraud/**", async () => {const partnerEmailDomain = extractEmailDomain(E2E_FRAUD_PARTNER.email)!;- const customer = randomCustomer({- emailDomain: partnerEmailDomain,- });+ const customer = {+ ...randomCustomer(),+ email: randomUniqueEmail(partnerEmailDomain),+ };await http.post<TrackLeadResponse>({path: "/track/lead",@@ -110,7 +119,10 @@ describe.concurrent("/fraud/**", async () => {const trackedClickId = clickResponse.data.clickId;- const customer = randomCustomer({ emailDomain: "google.com" });+ const customer = {+ ...randomCustomer(),+ email: randomUniqueEmail("google.com"),+ };await http.post<TrackLeadResponse>({path: "/track/lead",@@ -135,6 +147,52 @@ describe.concurrent("/fraud/**", async () => {});});+ test("FraudRuleType = customerEmailMatch (similar email match)", async () => {+ const clickLink = E2E_FRAUD_PARTNER.links.customerEmailMatch;+ const localPart = randomEmailLocalPart();++ const firstCustomer = {+ ...randomCustomer(),+ email: `${localPart}@gmail.com`,+ };++ const similarCustomer = {+ ...randomCustomer(),+ email: `${localPart.split("").join(".")}@gmail.com`,+ };++ for (const customer of [firstCustomer, similarCustomer]) {+ const clickResponse = await http.post<{ clickId: string }>({+ path: "/track/click",+ headers: { ...E2E_TRACK_CLICK_HEADERS },+ body: { domain: clickLink.domain, key: clickLink.key },+ });++ await http.post<TrackLeadResponse>({+ path: "/track/lead",+ body: {+ eventName: "Signup",+ clickId: clickResponse.data.clickId,+ customerId: customer.externalId,+ customerName: customer.name,+ customerEmail: customer.email,+ customerAvatar: customer.avatar,+ },+ });+ }++ await verifyFraudEvent({+ http,+ partner: E2E_FRAUD_PARTNER,+ customer: similarCustomer,+ ruleType: "customerEmailMatch",+ metadata: {+ matchType: CustomerEmailMatchType.SIMILAR_EMAIL_MATCH,+ matchedCustomerId: expect.any(String),+ },+ });+ });+test("FraudRuleType = customerEmailSuspiciousDomain", async () => {const clickLink = E2E_FRAUD_PARTNER.links.customerEmailSuspiciousDomain;