Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

improve the backfill script

Kiran K0d8a41eb6f74
diff --git a/apps/web/lib/actions/partners/bulk-reject-partner-applications.ts b/apps/web/lib/actions/partners/bulk-reject-partner-applications.ts
index 39eda5cc1a..1a7150c540 100644
--- a/apps/web/lib/actions/partners/bulk-reject-partner-applications.ts
+++ b/apps/web/lib/actions/partners/bulk-reject-partner-applications.ts
@@ -9,7 +9,6 @@ import { prisma } from "@/lib/prisma";
import { bulkRejectPartnersSchema } from "@/lib/zod/schemas/partners";
import { sendBatchEmail } from "@dub/email";
import PartnerApplicationRejected from "@dub/email/templates/partner-application-rejected";
-import { pluck } from "@dub/utils";
import {
ProgramApplicationStatus,
ProgramEnrollmentStatus,
@@ -51,30 +50,37 @@ export const bulkRejectPartnerApplicationsAction = authActionClient
return;
}
- const applicationIds = programEnrollments
- .map(({ applicationId }) => applicationId)
- .filter((id): id is string => Boolean(id));
-
const reviewedAt = new Date();
- await prisma.$transaction(async (tx) => {
- await tx.programEnrollment.updateMany({
- where: {
- id: {
- in: programEnrollments.map(({ id }) => id),
+ const rejectedEnrollmentIds = await prisma.$transaction(async (tx) => {
+ const transitionedIds = new Set<string>();
+
+ for (const { id } of programEnrollments) {
+ const { count } = await tx.programEnrollment.updateMany({
+ where: {
+ id,
+ status: ProgramEnrollmentStatus.pending,
},
- status: ProgramEnrollmentStatus.pending,
- },
- data: {
- status: ProgramEnrollmentStatus.rejected,
- clickRewardId: null,
- leadRewardId: null,
- saleRewardId: null,
- referralRewardId: null,
- customRewardId: null,
- discountId: null,
- },
- });
+ data: {
+ status: ProgramEnrollmentStatus.rejected,
+ clickRewardId: null,
+ leadRewardId: null,
+ saleRewardId: null,
+ referralRewardId: null,
+ customRewardId: null,
+ discountId: null,
+ },
+ });
+
+ if (count > 0) {
+ transitionedIds.add(id);
+ }
+ }
+
+ const applicationIds = programEnrollments
+ .filter(({ id }) => transitionedIds.has(id))
+ .map(({ applicationId }) => applicationId)
+ .filter((id): id is string => Boolean(id));
if (applicationIds.length > 0) {
await tx.programApplication.updateMany({
@@ -82,9 +88,6 @@ export const bulkRejectPartnerApplicationsAction = authActionClient
id: {
in: applicationIds,
},
- enrollment: {
- status: ProgramEnrollmentStatus.rejected,
- },
},
data: {
status: ProgramApplicationStatus.rejected,
@@ -95,23 +98,11 @@ export const bulkRejectPartnerApplicationsAction = authActionClient
},
});
}
- });
- // Find the rejected enrollments
- const updatedEnrollments = await prisma.programEnrollment.findMany({
- where: {
- id: {
- in: pluck(programEnrollments, "id"),
- },
- status: ProgramEnrollmentStatus.rejected,
- },
- select: {
- id: true,
- },
+ return transitionedIds;
});
- const rejectedEnrollmentIds = new Set(pluck(updatedEnrollments, "id"));
-
+ // Find the rejected enrollments
const rejectedEnrollments = programEnrollments.filter(({ id }) =>
rejectedEnrollmentIds.has(id),
);
diff --git a/apps/web/scripts/migrations/backfill-program-application-status.ts b/apps/web/scripts/migrations/backfill-program-application-status.ts
index d76b5752bf..6ca8ecaac8 100644
--- a/apps/web/scripts/migrations/backfill-program-application-status.ts
+++ b/apps/web/scripts/migrations/backfill-program-application-status.ts
@@ -1,3 +1,4 @@
+import { getAuditLogs } from "@/lib/api/audit-logs/get-audit-logs";
import { getApplicationStatusFromEnrollment } from "@/lib/partners/get-application-status-from-enrollment";
import { prisma } from "@/lib/prisma";
import { ProgramApplicationStatus } from "@prisma/client";
@@ -9,11 +10,63 @@ const BATCH_SIZE = 500;
// Max gap between an application's reviewedAt and the activity log written by the same review
const REVIEW_LOG_MATCH_WINDOW_MS = 10 * 60 * 1000;
+// Approve/reject wrote Tinybird audit logs until #3746 (2026-04-13) replaced them with ActivityLog rows
+const ACTIVITY_LOGS_INTRODUCED_AT = new Date("2026-04-14");
+
const REVIEW_LOG_ACTIONS = {
"partner_application.approved": ProgramApplicationStatus.approved,
"partner_application.rejected": ProgramApplicationStatus.rejected,
} as const;
+type ReviewLogAction = keyof typeof REVIEW_LOG_ACTIONS;
+
+function isReviewLogAction(action: string): action is ReviewLogAction {
+ return action in REVIEW_LOG_ACTIONS;
+}
+
+// Returns the decision of the review log closest to reviewedAt, within the match window
+function findClosestDecision(
+ logs: { action: string; createdAt: Date }[],
+ reviewedAt: Date,
+) {
+ let closest: { action: ReviewLogAction; gap: number } | null = null;
+
+ for (const log of logs) {
+ if (!isReviewLogAction(log.action)) {
+ continue;
+ }
+
+ const gap = Math.abs(log.createdAt.getTime() - reviewedAt.getTime());
+
+ if (gap <= REVIEW_LOG_MATCH_WINDOW_MS && (!closest || gap < closest.gap)) {
+ closest = {
+ action: log.action,
+ gap,
+ };
+ }
+ }
+
+ return closest ? REVIEW_LOG_ACTIONS[closest.action] : null;
+}
+
+// Tinybird returns DateTime64 as "YYYY-MM-DD HH:mm:ss.SSS" in UTC, without a timezone suffix
+function parseTinybirdTimestamp(timestamp: string) {
+ return new Date(
+ /[zZ]|[+-]\d{2}:?\d{2}$/.test(timestamp)
+ ? timestamp
+ : `${timestamp.replace(" ", "T")}Z`,
+ );
+}
+
+function getAuditLogPartnerId(targets: string) {
+ try {
+ const parsed = JSON.parse(targets) as { type?: string; id?: string }[];
+ return parsed.find(({ type }) => type === "partner")?.id ?? null;
+ } catch {
+ return null;
+ }
+}
+
// Pass 1: applications linked to an enrollment take their status and partnerId from it
async function backfillFromEnrollments() {
console.log("Pass 1: applications with an enrollment");
@@ -468,34 +521,16 @@ async function backfillFromActivityLogs() {
const unmatched: typeof candidates = [];
for (const candidate of candidates) {
- const reviewedAt = candidate.reviewedAt.getTime();
-
- let closest: { action: string; gap: number } | null = null;
-
- for (const log of logsByKey.get(
- `${candidate.programId}:${candidate.partnerId}`,
- ) ?? []) {
- const gap = Math.abs(log.createdAt.getTime() - reviewedAt);
-
- if (
- gap <= REVIEW_LOG_MATCH_WINDOW_MS &&
- (!closest || gap < closest.gap)
- ) {
- closest = {
- action: log.action,
- gap,
- };
- }
- }
+ const status = findClosestDecision(
+ logsByKey.get(`${candidate.programId}:${candidate.partnerId}`) ?? [],
+ candidate.reviewedAt,
+ );
- if (!closest) {
+ if (!status) {
unmatched.push(candidate);
continue;
}
- const status =
- REVIEW_LOG_ACTIONS[closest.action as keyof typeof REVIEW_LOG_ACTIONS];
-
idsByStatus[status].push(candidate.id);
}
@@ -546,6 +581,220 @@ async function backfillFromActivityLogs() {
console.table(totals);
}
+// Pass 5: reviewed applications from before ActivityLog existed take the decision from the
+// Tinybird audit log written by the same review (matched by programId, partnerId, and closest to reviewedAt).
+// dub_audit_logs has a 1-year TTL, so this has to run before the 2026-03-30+ logs expire.
+async function backfillFromAuditLogs() {
+ console.log(
+ "Pass 5: reviewed applications without an enrollment, from the Tinybird audit log",
+ );
+
+ const totals = {
+ scanned: 0,
+ approved: 0,
+ rejected: 0,
+ noPartnerId: 0,
+ noMatchingAuditLog: 0,
+ };
+
+ let cursor: string | undefined;
+
+ while (true) {
+ const applications = await prisma.programApplication.findMany({
+ where: {
+ enrollment: null,
+ status: ProgramApplicationStatus.pending,
+ reviewedAt: {
+ not: null,
+ lt: ACTIVITY_LOGS_INTRODUCED_AT,
+ },
+ },
+ select: {
+ id: true,
+ programId: true,
+ partnerId: true,
+ email: true,
+ reviewedAt: true,
+ },
+ orderBy: {
+ id: "asc",
+ },
+ take: BATCH_SIZE,
+ ...(cursor && {
+ skip: 1,
+ cursor: {
+ id: cursor,
+ },
+ }),
+ });
+
+ if (applications.length === 0) {
+ break;
+ }
+
+ totals.scanned += applications.length;
+ cursor = applications[applications.length - 1].id;
+
+ const partnerIdByEmail = await getPartnerIdsByEmail(
+ applications
+ .filter(({ partnerId }) => !partnerId)
+ .map(({ email }) => email),
+ );
+
+ const candidates = applications.flatMap((application) => {
+ const partnerId =
+ application.partnerId ??
+ partnerIdByEmail.get(application.email.toLowerCase());
+
+ if (!partnerId || !application.reviewedAt) {
+ totals.noPartnerId++;
+ return [];
+ }
+
+ return [
+ {
+ ...application,
+ partnerId,
+ reviewedAt: application.reviewedAt,
+ },
+ ];
+ });
+
+ const candidatesByProgramId = new Map<string, typeof candidates>();
+
+ for (const candidate of candidates) {
+ candidatesByProgramId.set(candidate.programId, [
+ ...(candidatesByProgramId.get(candidate.programId) ?? []),
+ candidate,
+ ]);
+ }
+
+ const programs = await prisma.program.findMany({
+ where: {
+ id: {
+ in: [...candidatesByProgramId.keys()],
+ },
+ },
+ select: {
+ id: true,
+ workspaceId: true,
+ },
+ });
+
+ const logsByKey = new Map<string, { action: string; createdAt: Date }[]>();
+
+ for (const program of programs) {
+ const programCandidates = candidatesByProgramId.get(program.id) ?? [];
+
+ const reviewedAtTimes = programCandidates.map(({ reviewedAt }) =>
+ reviewedAt.getTime(),
+ );
+
+ // TODO:
+ // We can add a custom pipe to do this
+ const auditLogs = await getAuditLogs({
+ workspaceId: program.workspaceId,
+ programId: program.id,
+ start: new Date(
+ Math.min(...reviewedAtTimes) - REVIEW_LOG_MATCH_WINDOW_MS,
+ ),
+ end: new Date(
+ Math.max(...reviewedAtTimes) + REVIEW_LOG_MATCH_WINDOW_MS,
+ ),
+ });
+
+ for (const auditLog of auditLogs) {
+ if (!isReviewLogAction(auditLog.action)) {
+ continue;
+ }
+
+ const partnerId = getAuditLogPartnerId(auditLog.targets);
+
+ if (!partnerId) {
+ continue;
+ }
+
+ const key = `${program.id}:${partnerId}`;
+
+ logsByKey.set(key, [
+ ...(logsByKey.get(key) ?? []),
+ {
+ action: auditLog.action,
+ createdAt: parseTinybirdTimestamp(auditLog.timestamp),
+ },
+ ]);
+ }
+ }
+
+ const idsByStatus: Record<ProgramApplicationStatus, string[]> = {
+ pending: [],
+ approved: [],
+ rejected: [],
+ };
+
+ const unmatched: typeof candidates = [];
+
+ for (const candidate of candidates) {
+ const status = findClosestDecision(
+ logsByKey.get(`${candidate.programId}:${candidate.partnerId}`) ?? [],
+ candidate.reviewedAt,
+ );
+
+ if (!status) {
+ unmatched.push(candidate);
+ continue;
+ }
+
+ idsByStatus[status].push(candidate.id);
+ }
+
+ totals.noMatchingAuditLog += unmatched.length;
+
+ if (unmatched.length > 0) {
+ console.log(
+ `${unmatched.length} reviewed applications have no matching audit log`,
+ );
+ console.table(unmatched.slice(0, 20));
+ }
+
+ for (const status of [
+ ProgramApplicationStatus.approved,
+ ProgramApplicationStatus.rejected,
+ ]) {
+ const ids = idsByStatus[status];
+
+ if (ids.length === 0) {
+ continue;
+ }
+
+ if (DRY_RUN) {
+ totals[status] += ids.length;
+ continue;
+ }
+
+ const { count } = await prisma.programApplication.updateMany({
+ where: {
+ id: {
+ in: ids,
+ },
+ status: ProgramApplicationStatus.pending,
+ },
+ data: {
+ status,
+ },
+ });
+
+ totals[status] += count;
+ }
+
+ console.log(
+ `${DRY_RUN ? "Would process" : "Processed"} ${applications.length} applications up to ${cursor}`,
+ );
+ }
+
+ console.table(totals);
+}
+
async function main() {
console.log(`DRY_RUN=${DRY_RUN} BATCH_SIZE=${BATCH_SIZE}`);
@@ -554,6 +803,7 @@ async function main() {
await backfillRejectedFromRejectionFields();
await backfillPartnerIdsByEmail();
await backfillFromActivityLogs();
+ await backfillFromAuditLogs();
console.log("Finished.");
}