Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Make merge accounts more scalable for `commissions` and `customers`

Steven Tey4b8bebb1f0c1
diff --git a/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.ts b/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.ts
index ee68e5b26b..6c54a798c0 100644
--- a/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.ts
+++ b/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.ts
@@ -1,4 +1,5 @@
import { handleAndReturnErrorResponse } from "@/lib/api/errors";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
import { prisma } from "@/lib/prisma";
import { log } from "@dub/utils";
@@ -29,7 +30,7 @@ export async function POST(req: Request) {
lt: subDays(new Date(), 90),
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (declinedProgramEnrollments.length === 0) {
diff --git a/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.ts b/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.ts
index 2db83e27d5..172cf75a15 100644
--- a/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.ts
+++ b/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.ts
@@ -1,4 +1,5 @@
import { handleAndReturnErrorResponse } from "@/lib/api/errors";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
import { prisma } from "@/lib/prisma";
import { log } from "@dub/utils";
@@ -35,7 +36,7 @@ export async function POST(req: Request) {
none: {},
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (rejectedProgramEnrollments.length === 0) {
diff --git a/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.ts b/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.ts
index c4281d6398..b6df9a7501 100644
--- a/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.ts
+++ b/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.ts
@@ -1,5 +1,6 @@
import { handleAndReturnErrorResponse } from "@/lib/api/errors";
import { bulkDeletePartners } from "@/lib/api/partners/bulk-delete-partners";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
import { prisma } from "@/lib/prisma";
import { log } from "@dub/utils";
@@ -32,7 +33,7 @@ export async function POST(req: Request) {
none: {},
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (partnersToDelete.length === 0) {
diff --git a/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.ts b/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.ts
index 440e120932..564517dd6f 100644
--- a/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.ts
+++ b/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.ts
@@ -4,6 +4,7 @@ import { linkCache } from "@/lib/api/links/cache";
import { includeProgramEnrollment } from "@/lib/api/links/include-program-enrollment";
import { includeTags } from "@/lib/api/links/include-tags";
import { syncTotalCommissions } from "@/lib/api/partners/sync-total-commissions";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { verifyQstashSignature } from "@/lib/cron/verify-qstash";
import { conn } from "@/lib/planetscale";
import { prisma } from "@/lib/prisma";
@@ -157,23 +158,35 @@ export async function POST(req: Request) {
},
};
- // update links, commissions, bounty submissions, and payouts
+ // update links and payouts
if (programIdsToTransfer.length > 0) {
- const [
- updatedLinksRes,
- updatedCustomersRes,
- updatedCommissionsRes,
- updatedPayoutsRes,
- ] = await Promise.all([
+ const [updatedLinksRes, updatedPayoutsRes] = await Promise.all([
prisma.link.updateMany(updateManyPayload),
- prisma.customer.updateMany(updateManyPayload),
- prisma.commission.updateMany(updateManyPayload),
prisma.payout.updateMany(updateManyPayload),
]);
console.log(
- `Updated ${updatedLinksRes.count} links, ${updatedCustomersRes.count} customers, ${updatedCommissionsRes.count} commissions, and ${updatedPayoutsRes.count} payouts`,
+ `Updated ${updatedLinksRes.count} links, and ${updatedPayoutsRes.count} payouts`,
);
+ // for commissions / customers, we need to update them in batches of PRISMA_UPDATEMANY_LIMIT cause there can be a lot of them
+ while (true) {
+ const { count } = await prisma.commission.updateMany({
+ ...updateManyPayload,
+ limit: PRISMA_UPDATEMANY_LIMIT,
+ });
+ console.log(`Updated ${count} commissions`);
+ if (count < PRISMA_UPDATEMANY_LIMIT) break;
+ }
+
+ while (true) {
+ const { count } = await prisma.customer.updateMany({
+ ...updateManyPayload,
+ limit: PRISMA_UPDATEMANY_LIMIT,
+ });
+ console.log(`Updated ${count} customers`);
+ if (count < PRISMA_UPDATEMANY_LIMIT) break;
+ }
+
// update discount codes, notification emails, messages, and partner comments
const [
updatedDiscountCodesRes,
diff --git a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts
index 72d245bc84..fd2a033497 100644
--- a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts
+++ b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts
@@ -1,3 +1,4 @@
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import { redis } from "@/lib/upstash";
import { LINK_CLICKED_WEBHOOK_WORKSPACES_REDIS_KEY } from "@/lib/webhook/click-webhook-workspaces";
@@ -64,7 +65,7 @@ export const cleanupRedundantLinkWebhookEntries = async () => {
in: nonLinkScopeWebhooks.map((webhook) => webhook.id),
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
const deleted = await prisma.linkWebhook.deleteMany({
where: {
diff --git a/apps/web/lib/api/fraud/hold-pending-commissions.ts b/apps/web/lib/api/fraud/hold-pending-commissions.ts
index 7276ea2428..5ba5158db7 100644
--- a/apps/web/lib/api/fraud/hold-pending-commissions.ts
+++ b/apps/web/lib/api/fraud/hold-pending-commissions.ts
@@ -1,4 +1,5 @@
import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import { chunk, groupBy } from "@dub/utils";
import { CommissionStatus, Prisma, ProgramEnrollment } from "@prisma/client";
@@ -60,7 +61,7 @@ export async function holdPendingCommissions(
},
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (pendingCommissions.length === 0) {
diff --git a/apps/web/lib/api/fraud/hold-processed-commissions.ts b/apps/web/lib/api/fraud/hold-processed-commissions.ts
index 5e93e62492..fcb5962213 100644
--- a/apps/web/lib/api/fraud/hold-processed-commissions.ts
+++ b/apps/web/lib/api/fraud/hold-processed-commissions.ts
@@ -1,4 +1,5 @@
import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { retallyPayoutsAmount } from "@/lib/payouts/retally-payouts-amount";
import { prisma } from "@/lib/prisma";
import { chunk, groupBy } from "@dub/utils";
@@ -72,7 +73,7 @@ export async function holdProcessedCommissions(
},
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (processedCommissions.length === 0) {
diff --git a/apps/web/lib/api/fraud/release-hold-commissions.ts b/apps/web/lib/api/fraud/release-hold-commissions.ts
index 959db0be41..c768b40e74 100644
--- a/apps/web/lib/api/fraud/release-hold-commissions.ts
+++ b/apps/web/lib/api/fraud/release-hold-commissions.ts
@@ -2,7 +2,7 @@ import { triggerAggregateDueCommissionsCronJob } from "@/lib/actions/partners/tr
import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";
import { syncTotalCommissions } from "@/lib/api/partners/sync-total-commissions";
import { executeWorkflows } from "@/lib/api/workflows/execute-workflows";
-import { qstash } from "@/lib/cron";
+import { PRISMA_UPDATEMANY_LIMIT, qstash } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
import {
@@ -129,7 +129,7 @@ export async function releaseHoldCommissions({
earnings: true,
status: true,
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (commissionsToRelease.length === 0) {
diff --git a/apps/web/lib/cron/index.ts b/apps/web/lib/cron/index.ts
index 7b9eb8fdc8..dece18a598 100644
--- a/apps/web/lib/cron/index.ts
+++ b/apps/web/lib/cron/index.ts
@@ -7,3 +7,6 @@ export const qstash = new Client({
// Default batch size for cron jobs that process records in batches
export const CRON_BATCH_SIZE = 100;
+
+// 250 is generally the best size limit for Prisma updateMany operations
+export const PRISMA_UPDATEMANY_LIMIT = 250;
diff --git a/apps/web/scripts/customers/beehiiv/fix-case-a-complex.ts b/apps/web/scripts/customers/beehiiv/fix-case-a-complex.ts
index 294479a8b8..a29b3f485c 100644
--- a/apps/web/scripts/customers/beehiiv/fix-case-a-complex.ts
+++ b/apps/web/scripts/customers/beehiiv/fix-case-a-complex.ts
@@ -1,4 +1,5 @@
import { createId } from "@/lib/api/create-id";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { retallyPayoutsAmount } from "@/lib/payouts/retally-payouts-amount";
import { prisma } from "@/lib/prisma";
import { groupBy, linkConstructorSimple } from "@dub/utils";
@@ -113,9 +114,9 @@ async function main() {
data: {
partnerId: link.actualPartnerId!,
},
- limit: 250,
+ limit: PRISMA_UPDATEMANY_LIMIT,
});
- if (updatedCommissions.count < 250) {
+ if (updatedCommissions.count < PRISMA_UPDATEMANY_LIMIT) {
break;
}
console.log(
diff --git a/apps/web/scripts/migrations/backfill-hold-pending-commissions.ts b/apps/web/scripts/migrations/backfill-hold-pending-commissions.ts
index 571e386fa3..093bf7614b 100644
--- a/apps/web/scripts/migrations/backfill-hold-pending-commissions.ts
+++ b/apps/web/scripts/migrations/backfill-hold-pending-commissions.ts
@@ -2,6 +2,7 @@ import {
CUSTOMER_LEVEL_FRAUD_RULES,
PARTNER_LEVEL_FRAUD_RULES,
} from "@/lib/api/fraud/constants";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import { groupBy } from "@dub/utils";
import { CommissionStatus, FraudEventStatus, Prisma } from "@prisma/client";
@@ -130,7 +131,7 @@ async function main() {
},
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (commissions.length === 0) {
diff --git a/apps/web/scripts/migrations/backfill-hold-processed-commissions.ts b/apps/web/scripts/migrations/backfill-hold-processed-commissions.ts
index 7d4578fe89..5d3d76639d 100644
--- a/apps/web/scripts/migrations/backfill-hold-processed-commissions.ts
+++ b/apps/web/scripts/migrations/backfill-hold-processed-commissions.ts
@@ -2,6 +2,7 @@ import {
CUSTOMER_LEVEL_FRAUD_RULES,
PARTNER_LEVEL_FRAUD_RULES,
} from "@/lib/api/fraud/constants";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import { groupBy } from "@dub/utils";
import {
@@ -142,7 +143,7 @@ async function main() {
},
},
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
if (commissions.length === 0) {
diff --git a/apps/web/scripts/partners/update-payout-dates.ts b/apps/web/scripts/partners/update-payout-dates.ts
index a06b18fff9..e585d73efd 100644
--- a/apps/web/scripts/partners/update-payout-dates.ts
+++ b/apps/web/scripts/partners/update-payout-dates.ts
@@ -1,3 +1,4 @@
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import "dotenv-flow/config";
@@ -15,7 +16,7 @@ async function main() {
orderBy: {
createdAt: "desc",
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
});
console.table(payoutsToUpdate);
diff --git a/apps/web/scripts/programs/delete-program-enrollments.ts b/apps/web/scripts/programs/delete-program-enrollments.ts
index 34c35628ef..796021e233 100644
--- a/apps/web/scripts/programs/delete-program-enrollments.ts
+++ b/apps/web/scripts/programs/delete-program-enrollments.ts
@@ -1,4 +1,5 @@
import { bulkDeleteLinks } from "@/lib/api/links/bulk-delete-links";
+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";
import { prisma } from "@/lib/prisma";
import "dotenv-flow/config";
@@ -11,7 +12,7 @@ async function main() {
programId,
totalLeads: 0,
},
- take: 250,
+ take: PRISMA_UPDATEMANY_LIMIT,
include: {
links: true,
},