The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
improve the backfill script
Kiran K0d8a41eb6f74
diff --git a/apps/web/lib/actions/partners/bulk-reject-partner-applications.ts b/apps/web/lib/actions/partners/bulk-reject-partner-applications.tsindex 39eda5cc1a..1a7150c540 100644--- a/apps/web/lib/actions/partners/bulk-reject-partner-applications.ts+++ b/apps/web/lib/actions/partners/bulk-reject-partner-applications.ts@@ -9,7 +9,6 @@ import { prisma } from "@/lib/prisma";import { bulkRejectPartnersSchema } from "@/lib/zod/schemas/partners";import { sendBatchEmail } from "@dub/email";import PartnerApplicationRejected from "@dub/email/templates/partner-application-rejected";-import { pluck } from "@dub/utils";import {ProgramApplicationStatus,ProgramEnrollmentStatus,@@ -51,30 +50,37 @@ export const bulkRejectPartnerApplicationsAction = authActionClientreturn;}- const applicationIds = programEnrollments- .map(({ applicationId }) => applicationId)- .filter((id): id is string => Boolean(id));-const reviewedAt = new Date();- await prisma.$transaction(async (tx) => {- await tx.programEnrollment.updateMany({- where: {- id: {- in: programEnrollments.map(({ id }) => id),+ const rejectedEnrollmentIds = await prisma.$transaction(async (tx) => {+ const transitionedIds = new Set<string>();++ for (const { id } of programEnrollments) {+ const { count } = await tx.programEnrollment.updateMany({+ where: {+ id,+ status: ProgramEnrollmentStatus.pending,},- status: ProgramEnrollmentStatus.pending,- },- data: {- status: ProgramEnrollmentStatus.rejected,- clickRewardId: null,- leadRewardId: null,- saleRewardId: null,- referralRewardId: null,- customRewardId: null,- discountId: null,- },- });+ data: {+ status: ProgramEnrollmentStatus.rejected,+ clickRewardId: null,+ leadRewardId: null,+ saleRewardId: null,+ referralRewardId: null,+ customRewardId: null,+ discountId: null,+ },+ });++ if (count > 0) {+ transitionedIds.add(id);+ }+ }++ const applicationIds = programEnrollments+ .filter(({ id }) => transitionedIds.has(id))+ .map(({ applicationId }) => applicationId)+ .filter((id): id is string => Boolean(id));if (applicationIds.length > 0) {await tx.programApplication.updateMany({@@ -82,9 +88,6 @@ export const bulkRejectPartnerApplicationsAction = authActionClientid: {in: applicationIds,},- enrollment: {- status: ProgramEnrollmentStatus.rejected,- },},data: {status: ProgramApplicationStatus.rejected,@@ -95,23 +98,11 @@ export const bulkRejectPartnerApplicationsAction = authActionClient},});}- });- // Find the rejected enrollments- const updatedEnrollments = await prisma.programEnrollment.findMany({- where: {- id: {- in: pluck(programEnrollments, "id"),- },- status: ProgramEnrollmentStatus.rejected,- },- select: {- id: true,- },+ return transitionedIds;});- const rejectedEnrollmentIds = new Set(pluck(updatedEnrollments, "id"));-+ // Find the rejected enrollmentsconst rejectedEnrollments = programEnrollments.filter(({ id }) =>rejectedEnrollmentIds.has(id),);diff --git a/apps/web/scripts/migrations/backfill-program-application-status.ts b/apps/web/scripts/migrations/backfill-program-application-status.tsindex d76b5752bf..6ca8ecaac8 100644--- a/apps/web/scripts/migrations/backfill-program-application-status.ts+++ b/apps/web/scripts/migrations/backfill-program-application-status.ts@@ -1,3 +1,4 @@+import { getAuditLogs } from "@/lib/api/audit-logs/get-audit-logs";import { getApplicationStatusFromEnrollment } from "@/lib/partners/get-application-status-from-enrollment";import { prisma } from "@/lib/prisma";import { ProgramApplicationStatus } from "@prisma/client";@@ -9,11 +10,63 @@ const BATCH_SIZE = 500;// Max gap between an application's reviewedAt and the activity log written by the same reviewconst REVIEW_LOG_MATCH_WINDOW_MS = 10 * 60 * 1000;+// Approve/reject wrote Tinybird audit logs until #3746 (2026-04-13) replaced them with ActivityLog rows+const ACTIVITY_LOGS_INTRODUCED_AT = new Date("2026-04-14");+const REVIEW_LOG_ACTIONS = {"partner_application.approved": ProgramApplicationStatus.approved,"partner_application.rejected": ProgramApplicationStatus.rejected,} as const;+type ReviewLogAction = keyof typeof REVIEW_LOG_ACTIONS;++function isReviewLogAction(action: string): action is ReviewLogAction {+ return action in REVIEW_LOG_ACTIONS;+}++// Returns the decision of the review log closest to reviewedAt, within the match window+function findClosestDecision(+ logs: { action: string; createdAt: Date }[],+ reviewedAt: Date,+) {+ let closest: { action: ReviewLogAction; gap: number } | null = null;++ for (const log of logs) {+ if (!isReviewLogAction(log.action)) {+ continue;+ }++ const gap = Math.abs(log.createdAt.getTime() - reviewedAt.getTime());++ if (gap <= REVIEW_LOG_MATCH_WINDOW_MS && (!closest || gap < closest.gap)) {+ closest = {+ action: log.action,+ gap,+ };+ }+ }++ return closest ? REVIEW_LOG_ACTIONS[closest.action] : null;+}++// Tinybird returns DateTime64 as "YYYY-MM-DD HH:mm:ss.SSS" in UTC, without a timezone suffix+function parseTinybirdTimestamp(timestamp: string) {+ return new Date(+ /[zZ]|[+-]\d{2}:?\d{2}$/.test(timestamp)+ ? timestamp+ : `${timestamp.replace(" ", "T")}Z`,+ );+}++function getAuditLogPartnerId(targets: string) {+ try {+ const parsed = JSON.parse(targets) as { type?: string; id?: string }[];+ return parsed.find(({ type }) => type === "partner")?.id ?? null;+ } catch {+ return null;+ }+}+// Pass 1: applications linked to an enrollment take their status and partnerId from itasync function backfillFromEnrollments() {console.log("Pass 1: applications with an enrollment");@@ -468,34 +521,16 @@ async function backfillFromActivityLogs() {const unmatched: typeof candidates = [];for (const candidate of candidates) {- const reviewedAt = candidate.reviewedAt.getTime();-- let closest: { action: string; gap: number } | null = null;-- for (const log of logsByKey.get(- `${candidate.programId}:${candidate.partnerId}`,- ) ?? []) {- const gap = Math.abs(log.createdAt.getTime() - reviewedAt);-- if (- gap <= REVIEW_LOG_MATCH_WINDOW_MS &&- (!closest || gap < closest.gap)- ) {- closest = {- action: log.action,- gap,- };- }- }+ const status = findClosestDecision(+ logsByKey.get(`${candidate.programId}:${candidate.partnerId}`) ?? [],+ candidate.reviewedAt,+ );- if (!closest) {+ if (!status) {unmatched.push(candidate);continue;}- const status =- REVIEW_LOG_ACTIONS[closest.action as keyof typeof REVIEW_LOG_ACTIONS];-idsByStatus[status].push(candidate.id);}@@ -546,6 +581,220 @@ async function backfillFromActivityLogs() {console.table(totals);}+// Pass 5: reviewed applications from before ActivityLog existed take the decision from the+// Tinybird audit log written by the same review (matched by programId, partnerId, and closest to reviewedAt).+// dub_audit_logs has a 1-year TTL, so this has to run before the 2026-03-30+ logs expire.+async function backfillFromAuditLogs() {+ console.log(+ "Pass 5: reviewed applications without an enrollment, from the Tinybird audit log",+ );++ const totals = {+ scanned: 0,+ approved: 0,+ rejected: 0,+ noPartnerId: 0,+ noMatchingAuditLog: 0,+ };++ let cursor: string | undefined;++ while (true) {+ const applications = await prisma.programApplication.findMany({+ where: {+ enrollment: null,+ status: ProgramApplicationStatus.pending,+ reviewedAt: {+ not: null,+ lt: ACTIVITY_LOGS_INTRODUCED_AT,+ },+ },+ select: {+ id: true,+ programId: true,+ partnerId: true,+ email: true,+ reviewedAt: true,+ },+ orderBy: {+ id: "asc",+ },+ take: BATCH_SIZE,+ ...(cursor && {+ skip: 1,+ cursor: {+ id: cursor,+ },+ }),+ });++ if (applications.length === 0) {+ break;+ }++ totals.scanned += applications.length;+ cursor = applications[applications.length - 1].id;++ const partnerIdByEmail = await getPartnerIdsByEmail(+ applications+ .filter(({ partnerId }) => !partnerId)+ .map(({ email }) => email),+ );++ const candidates = applications.flatMap((application) => {+ const partnerId =+ application.partnerId ??+ partnerIdByEmail.get(application.email.toLowerCase());++ if (!partnerId || !application.reviewedAt) {+ totals.noPartnerId++;+ return [];+ }++ return [+ {+ ...application,+ partnerId,+ reviewedAt: application.reviewedAt,+ },+ ];+ });++ const candidatesByProgramId = new Map<string, typeof candidates>();++ for (const candidate of candidates) {+ candidatesByProgramId.set(candidate.programId, [+ ...(candidatesByProgramId.get(candidate.programId) ?? []),+ candidate,+ ]);+ }++ const programs = await prisma.program.findMany({+ where: {+ id: {+ in: [...candidatesByProgramId.keys()],+ },+ },+ select: {+ id: true,+ workspaceId: true,+ },+ });++ const logsByKey = new Map<string, { action: string; createdAt: Date }[]>();++ for (const program of programs) {+ const programCandidates = candidatesByProgramId.get(program.id) ?? [];++ const reviewedAtTimes = programCandidates.map(({ reviewedAt }) =>+ reviewedAt.getTime(),+ );++ // TODO:+ // We can add a custom pipe to do this+ const auditLogs = await getAuditLogs({+ workspaceId: program.workspaceId,+ programId: program.id,+ start: new Date(+ Math.min(...reviewedAtTimes) - REVIEW_LOG_MATCH_WINDOW_MS,+ ),+ end: new Date(+ Math.max(...reviewedAtTimes) + REVIEW_LOG_MATCH_WINDOW_MS,+ ),+ });++ for (const auditLog of auditLogs) {+ if (!isReviewLogAction(auditLog.action)) {+ continue;+ }++ const partnerId = getAuditLogPartnerId(auditLog.targets);++ if (!partnerId) {+ continue;+ }++ const key = `${program.id}:${partnerId}`;++ logsByKey.set(key, [+ ...(logsByKey.get(key) ?? []),+ {+ action: auditLog.action,+ createdAt: parseTinybirdTimestamp(auditLog.timestamp),+ },+ ]);+ }+ }++ const idsByStatus: Record<ProgramApplicationStatus, string[]> = {+ pending: [],+ approved: [],+ rejected: [],+ };++ const unmatched: typeof candidates = [];++ for (const candidate of candidates) {+ const status = findClosestDecision(+ logsByKey.get(`${candidate.programId}:${candidate.partnerId}`) ?? [],+ candidate.reviewedAt,+ );++ if (!status) {+ unmatched.push(candidate);+ continue;+ }++ idsByStatus[status].push(candidate.id);+ }++ totals.noMatchingAuditLog += unmatched.length;++ if (unmatched.length > 0) {+ console.log(+ `${unmatched.length} reviewed applications have no matching audit log`,+ );+ console.table(unmatched.slice(0, 20));+ }++ for (const status of [+ ProgramApplicationStatus.approved,+ ProgramApplicationStatus.rejected,+ ]) {+ const ids = idsByStatus[status];++ if (ids.length === 0) {+ continue;+ }++ if (DRY_RUN) {+ totals[status] += ids.length;+ continue;+ }++ const { count } = await prisma.programApplication.updateMany({+ where: {+ id: {+ in: ids,+ },+ status: ProgramApplicationStatus.pending,+ },+ data: {+ status,+ },+ });++ totals[status] += count;+ }++ console.log(+ `${DRY_RUN ? "Would process" : "Processed"} ${applications.length} applications up to ${cursor}`,+ );+ }++ console.table(totals);+}+async function main() {console.log(`DRY_RUN=${DRY_RUN} BATCH_SIZE=${BATCH_SIZE}`);@@ -554,6 +803,7 @@ async function main() {await backfillRejectedFromRejectionFields();await backfillPartnerIdsByEmail();await backfillFromActivityLogs();+ await backfillFromAuditLogs();console.log("Finished.");}