The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Improve the backfill script
Kiran K3feac0aa9f1d
diff --git a/apps/web/scripts/migrations/backfill-program-application-status.ts b/apps/web/scripts/migrations/backfill-program-application-status.tsindex 1c834bde57..d76b5752bf 100644--- a/apps/web/scripts/migrations/backfill-program-application-status.ts+++ b/apps/web/scripts/migrations/backfill-program-application-status.ts@@ -1,72 +1,44 @@import { getApplicationStatusFromEnrollment } from "@/lib/partners/get-application-status-from-enrollment";import { prisma } from "@/lib/prisma";-import {- ProgramApplication,- ProgramApplicationStatus,- ProgramEnrollment,-} from "@prisma/client";+import { ProgramApplicationStatus } from "@prisma/client";import "dotenv-flow/config";const DRY_RUN = true;const BATCH_SIZE = 500;-// ProgramApplication.reviewedAt was introduced on this date (#3664) without a backfill-const REVIEWED_AT_INTRODUCED_AT = new Date("2026-03-30");--function getApplicationStatus({- application,- enrollment,- matchedPartnerId,-}: {- application: Pick<ProgramApplication, "reviewedAt">;- enrollment: Pick<ProgramEnrollment, "status"> | null;- matchedPartnerId: string | null;-}): ProgramApplicationStatus {- if (!enrollment) {- if (application.reviewedAt) {- return ProgramApplicationStatus.rejected;- }+// Max gap between an application's reviewedAt and the activity log written by the same review+const REVIEW_LOG_MATCH_WINDOW_MS = 10 * 60 * 1000;- // A partner signed up with this email, so the application was converted into an- // enrollment that has since been removed (e.g. by the rejected-applications cleanup cron)- return matchedPartnerId- ? ProgramApplicationStatus.rejected- : ProgramApplicationStatus.pending;- }+const REVIEW_LOG_ACTIONS = {+ "partner_application.approved": ProgramApplicationStatus.approved,+ "partner_application.rejected": ProgramApplicationStatus.rejected,+} as const;- return getApplicationStatusFromEnrollment(enrollment.status);-}+// Pass 1: applications linked to an enrollment take their status and partnerId from it+async function backfillFromEnrollments() {+ console.log("Pass 1: applications with an enrollment");-async function main() {- console.log(`DRY_RUN=${DRY_RUN} BATCH_SIZE=${BATCH_SIZE}`);-- const totals: Record<- | ProgramApplicationStatus- | "partnerIds"- | "inferredRejected"- | "inferredRejectedSinceReviewedAt",- number- > = {+ const totals = {+ scanned: 0,pending: 0,approved: 0,rejected: 0,partnerIds: 0,- inferredRejected: 0,- inferredRejectedSinceReviewedAt: 0,};- let totalScanned = 0;let cursor: string | undefined;while (true) {const applications = await prisma.programApplication.findMany({+ where: {+ enrollment: {+ isNot: null,+ },+ },select: {id: true,- email: true,partnerId: true,status: true,- reviewedAt: true,- createdAt: true,enrollment: {select: {partnerId: true,@@ -90,171 +62,500 @@ async function main() {break;}- totalScanned += applications.length;+ totals.scanned += applications.length;cursor = applications[applications.length - 1].id;- const unreviewedEmails = [- ...new Set(- applications- .filter(({ enrollment, reviewedAt }) => !enrollment && !reviewedAt)- .map(({ email }) => email.toLowerCase()),- ),- ];-- const users =- unreviewedEmails.length > 0- ? await prisma.user.findMany({- where: {- email: {- in: unreviewedEmails,- },- partners: {- some: {},- },- },- select: {- email: true,- partners: {- select: {- partnerId: true,- },- take: 1,- },- },- })- : [];+ const idsByStatus: Record<ProgramApplicationStatus, string[]> = {+ pending: [],+ approved: [],+ rejected: [],+ };- const partnerIdByEmail = new Map(- users- .filter((user) => user.email && user.partners.length > 0)- .map((user) => [user.email!.toLowerCase(), user.partners[0].partnerId]),- );+ const partnerIdUpdates: { id: string; partnerId: string }[] = [];- const changes = applications.map((application) => {- const matchedPartnerId = application.enrollment- ? null- : partnerIdByEmail.get(application.email.toLowerCase()) ?? null;+ for (const application of applications) {+ const { enrollment } = application;- const status = getApplicationStatus({- application,- enrollment: application.enrollment,- matchedPartnerId,- });+ if (!enrollment) {+ continue;+ }- const partnerId =- application.enrollment?.partnerId ?? matchedPartnerId ?? null;+ const status = getApplicationStatusFromEnrollment(enrollment.status);- const updateStatus =+ // If status mismatch, update the status+ if (application.status === ProgramApplicationStatus.pending &&- status !== ProgramApplicationStatus.pending;-- return {- id: application.id,- enrollmentStatus: application.enrollment?.status ?? null,- reviewedAt: application.reviewedAt,- createdAt: application.createdAt,- currentStatus: application.status,- status,- updateStatus,- inferredRejected: Boolean(- updateStatus && !application.reviewedAt && matchedPartnerId,- ),- currentPartnerId: application.partnerId,- partnerId,- updatePartnerId: Boolean(- partnerId && application.partnerId !== partnerId,- ),- };- });+ status !== ProgramApplicationStatus.pending+ ) {+ idsByStatus[status].push(application.id);+ }- console.table(changes.slice(0, 20));+ // If partnerId is missing, set it from the enrollment+ if (!application.partnerId) {+ partnerIdUpdates.push({+ id: application.id,+ partnerId: enrollment.partnerId,+ });+ }+ }- const idsByStatus: Record<ProgramApplicationStatus, string[]> = {- pending: [],- approved: [],- rejected: [],- };+ for (const status of Object.values(ProgramApplicationStatus)) {+ const ids = idsByStatus[status];- for (const change of changes) {- if (change.updateStatus) {- idsByStatus[change.status].push(change.id);+ if (ids.length === 0) {+ continue;}- }- const partnerIdUpdates = changes.filter((change) => change.updatePartnerId);+ if (DRY_RUN) {+ totals[status] += ids.length;+ continue;+ }- const inferredRejected = changes.filter(- (change) => change.inferredRejected,- );+ const { count } = await prisma.programApplication.updateMany({+ where: {+ id: {+ in: ids,+ },+ status: ProgramApplicationStatus.pending,+ },+ data: {+ status,+ },+ });- if (inferredRejected.length > 0) {- console.log(- `Inferred ${inferredRejected.length} rejected applications from matching partner emails`,- );- console.table(inferredRejected.slice(0, 20));+ totals[status] += count;}- totals.inferredRejected += inferredRejected.length;- totals.inferredRejectedSinceReviewedAt += inferredRejected.filter(- (change) => change.createdAt >= REVIEWED_AT_INTRODUCED_AT,- ).length;-- if (DRY_RUN) {- for (const status of Object.values(ProgramApplicationStatus)) {- totals[status] += idsByStatus[status].length;+ if (partnerIdUpdates.length > 0) {+ if (!DRY_RUN) {+ await prisma.$transaction(+ partnerIdUpdates.map(({ id, partnerId }) =>+ prisma.programApplication.update({+ where: {+ id,+ },+ data: {+ partnerId,+ },+ }),+ ),+ );}totals.partnerIds += partnerIdUpdates.length;- } else {- for (const status of Object.values(ProgramApplicationStatus)) {- const ids = idsByStatus[status];+ }- if (ids.length === 0) {- continue;- }+ console.log(+ `${DRY_RUN ? "Would process" : "Processed"} ${applications.length} applications up to ${cursor}`,+ );+ }- const { count } = await prisma.programApplication.updateMany({- where: {- id: {- in: ids,+ console.table(totals);+}++// Pass 2: applications without an enrollment that carry a rejection reason or note were rejected+async function backfillRejectedFromRejectionFields() {+ console.log(+ "Pass 2: applications without an enrollment and a rejection reason or note",+ );++ const totals = {+ scanned: 0,+ rejected: 0,+ };++ let cursor: string | undefined;++ while (true) {+ const applications = await prisma.programApplication.findMany({+ where: {+ enrollment: null,+ status: ProgramApplicationStatus.pending,+ OR: [+ {+ rejectionReason: {+ not: null,},- status: ProgramApplicationStatus.pending,},- data: {- status,+ {+ rejectionNote: {+ not: null,+ },},- });+ ],+ },+ select: {+ id: true,+ rejectionReason: true,+ rejectionNote: true,+ reviewedAt: true,+ },+ orderBy: {+ id: "asc",+ },+ take: BATCH_SIZE,+ ...(cursor && {+ skip: 1,+ cursor: {+ id: cursor,+ },+ }),+ });- totals[status] += count;- }+ if (applications.length === 0) {+ break;+ }++ totals.scanned += applications.length;+ cursor = applications[applications.length - 1].id;++ console.table(applications.slice(0, 20));++ if (DRY_RUN) {+ totals.rejected += applications.length;+ } else {+ const { count } = await prisma.programApplication.updateMany({+ where: {+ id: {+ in: applications.map(({ id }) => id),+ },+ status: ProgramApplicationStatus.pending,+ },+ data: {+ status: ProgramApplicationStatus.rejected,+ },+ });++ totals.rejected += count;+ }++ console.log(+ `${DRY_RUN ? "Would process" : "Processed"} ${applications.length} applications up to ${cursor}`,+ );+ }++ console.table(totals);+}++// Match on the registered user's email (verified at login), not Partner.email,+// which importers and the partners API can set on partners nobody has claimed+async function getPartnerIdsByEmail(emails: string[]) {+ const uniqueEmails = [...new Set(emails.map((email) => email.toLowerCase()))];++ if (uniqueEmails.length === 0) {+ return new Map<string, string>();+ }++ const users = await prisma.user.findMany({+ where: {+ email: {+ in: uniqueEmails,+ },+ partners: {+ some: {},+ },+ },+ select: {+ email: true,+ partners: {+ select: {+ partnerId: true,+ },+ take: 1,+ },+ },+ });++ return new Map(+ users+ .filter((user) => user.email && user.partners.length > 0)+ .map((user) => [user.email!.toLowerCase(), user.partners[0].partnerId]),+ );+}++// Pass 3: applications without an enrollment get their partnerId from a registered partner with the same email+async function backfillPartnerIdsByEmail() {+ console.log("Pass 3: partnerId for applications without an enrollment");++ const totals = {+ scanned: 0,+ partnerIds: 0,+ };++ let cursor: string | undefined;++ while (true) {+ const applications = await prisma.programApplication.findMany({+ where: {+ enrollment: null,+ partnerId: null,+ },+ select: {+ id: true,+ email: true,+ },+ orderBy: {+ id: "asc",+ },+ take: BATCH_SIZE,+ ...(cursor && {+ skip: 1,+ cursor: {+ id: cursor,+ },+ }),+ });- if (partnerIdUpdates.length > 0) {+ if (applications.length === 0) {+ break;+ }++ totals.scanned += applications.length;+ cursor = applications[applications.length - 1].id;++ const partnerIdByEmail = await getPartnerIdsByEmail(+ applications.map(({ email }) => email),+ );++ const partnerIdUpdates = applications.flatMap(({ id, email }) => {+ const partnerId = partnerIdByEmail.get(email.toLowerCase());+ return partnerId ? [{ id, email, partnerId }] : [];+ });++ if (partnerIdUpdates.length > 0) {+ console.table(partnerIdUpdates.slice(0, 20));++ if (!DRY_RUN) {await prisma.$transaction(- partnerIdUpdates.map((change) =>+ partnerIdUpdates.map(({ id, partnerId }) =>prisma.programApplication.update({where: {- id: change.id,+ id,},data: {- partnerId: change.partnerId,+ partnerId,},}),),);-- totals.partnerIds += partnerIdUpdates.length;}++ totals.partnerIds += partnerIdUpdates.length;}console.log(- `${DRY_RUN ? "Would process" : "Processed"} ${applications.length} applications up to ${cursor} (scanned=${totalScanned})`,+ `${DRY_RUN ? "Would process" : "Processed"} ${applications.length} applications up to ${cursor}`,);}+ console.table(totals);+}++// Pass 4: reviewed applications without an enrollment take the decision from the activity log+// written by the same review (matched by programId, partnerId, and closest to reviewedAt)+async function backfillFromActivityLogs() {console.log(- `Finished. scanned=${totalScanned} ${DRY_RUN ? "would-update" : "updated"}:`,+ "Pass 4: reviewed applications without an enrollment, from the activity log",);++ const totals = {+ scanned: 0,+ approved: 0,+ rejected: 0,+ noPartnerId: 0,+ noMatchingLog: 0,+ };++ let cursor: string | undefined;++ while (true) {+ const applications = await prisma.programApplication.findMany({+ where: {+ enrollment: null,+ status: ProgramApplicationStatus.pending,+ reviewedAt: {+ not: null,+ },+ },+ select: {+ id: true,+ programId: true,+ partnerId: true,+ email: true,+ reviewedAt: true,+ },+ orderBy: {+ id: "asc",+ },+ take: BATCH_SIZE,+ ...(cursor && {+ skip: 1,+ cursor: {+ id: cursor,+ },+ }),+ });++ if (applications.length === 0) {+ break;+ }++ totals.scanned += applications.length;+ cursor = applications[applications.length - 1].id;++ const partnerIdByEmail = await getPartnerIdsByEmail(+ applications+ .filter(({ partnerId }) => !partnerId)+ .map(({ email }) => email),+ );++ const candidates = applications.flatMap((application) => {+ const partnerId =+ application.partnerId ??+ partnerIdByEmail.get(application.email.toLowerCase());++ if (!partnerId || !application.reviewedAt) {+ totals.noPartnerId++;+ return [];+ }++ return [+ {+ ...application,+ partnerId,+ reviewedAt: application.reviewedAt,+ },+ ];+ });++ const logs =+ candidates.length > 0+ ? await prisma.activityLog.findMany({+ where: {+ resourceType: "partner",+ resourceId: {+ in: [...new Set(candidates.map(({ partnerId }) => partnerId))],+ },+ programId: {+ in: [...new Set(candidates.map(({ programId }) => programId))],+ },+ action: {+ in: Object.keys(REVIEW_LOG_ACTIONS),+ },+ },+ select: {+ programId: true,+ resourceId: true,+ action: true,+ createdAt: true,+ },+ })+ : [];++ const logsByKey = new Map<string, typeof logs>();++ for (const log of logs) {+ const key = `${log.programId}:${log.resourceId}`;+ logsByKey.set(key, [...(logsByKey.get(key) ?? []), log]);+ }++ const idsByStatus: Record<ProgramApplicationStatus, string[]> = {+ pending: [],+ approved: [],+ rejected: [],+ };++ const unmatched: typeof candidates = [];++ for (const candidate of candidates) {+ const reviewedAt = candidate.reviewedAt.getTime();++ let closest: { action: string; gap: number } | null = null;++ for (const log of logsByKey.get(+ `${candidate.programId}:${candidate.partnerId}`,+ ) ?? []) {+ const gap = Math.abs(log.createdAt.getTime() - reviewedAt);++ if (+ gap <= REVIEW_LOG_MATCH_WINDOW_MS &&+ (!closest || gap < closest.gap)+ ) {+ closest = {+ action: log.action,+ gap,+ };+ }+ }++ if (!closest) {+ unmatched.push(candidate);+ continue;+ }++ const status =+ REVIEW_LOG_ACTIONS[closest.action as keyof typeof REVIEW_LOG_ACTIONS];++ idsByStatus[status].push(candidate.id);+ }++ totals.noMatchingLog += unmatched.length;++ if (unmatched.length > 0) {+ console.log(+ `${unmatched.length} reviewed applications have no matching activity log`,+ );+ console.table(unmatched.slice(0, 20));+ }++ for (const status of [+ ProgramApplicationStatus.approved,+ ProgramApplicationStatus.rejected,+ ]) {+ const ids = idsByStatus[status];++ if (ids.length === 0) {+ continue;+ }++ if (DRY_RUN) {+ totals[status] += ids.length;+ continue;+ }++ const { count } = await prisma.programApplication.updateMany({+ where: {+ id: {+ in: ids,+ },+ status: ProgramApplicationStatus.pending,+ },+ data: {+ status,+ },+ });++ totals[status] += count;+ }++ console.log(+ `${DRY_RUN ? "Would process" : "Processed"} ${applications.length} applications up to ${cursor}`,+ );+ }+console.table(totals);}+async function main() {+ console.log(`DRY_RUN=${DRY_RUN} BATCH_SIZE=${BATCH_SIZE}`);++ // In order:+ await backfillFromEnrollments();+ await backfillRejectedFromRejectionFields();+ await backfillPartnerIdsByEmail();+ await backfillFromActivityLogs();++ console.log("Finished.");+}+main();