The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Make merge accounts more scalable for `commissions` and `customers`
Steven Tey4b8bebb1f0c1
diff --git a/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.ts b/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.tsindex ee68e5b26b..6c54a798c0 100644--- a/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.ts+++ b/apps/web/app/(ee)/api/cron/cleanup/declined-invites/route.ts@@ -1,4 +1,5 @@import { handleAndReturnErrorResponse } from "@/lib/api/errors";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { verifyQstashSignature } from "@/lib/cron/verify-qstash";import { prisma } from "@/lib/prisma";import { log } from "@dub/utils";@@ -29,7 +30,7 @@ export async function POST(req: Request) {lt: subDays(new Date(), 90),},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (declinedProgramEnrollments.length === 0) {diff --git a/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.ts b/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.tsindex 2db83e27d5..172cf75a15 100644--- a/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.ts+++ b/apps/web/app/(ee)/api/cron/cleanup/rejected-applications/route.ts@@ -1,4 +1,5 @@import { handleAndReturnErrorResponse } from "@/lib/api/errors";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { verifyQstashSignature } from "@/lib/cron/verify-qstash";import { prisma } from "@/lib/prisma";import { log } from "@dub/utils";@@ -35,7 +36,7 @@ export async function POST(req: Request) {none: {},},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (rejectedProgramEnrollments.length === 0) {diff --git a/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.ts b/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.tsindex c4281d6398..b6df9a7501 100644--- a/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.ts+++ b/apps/web/app/(ee)/api/cron/cleanup/unenrolled-partners/route.ts@@ -1,5 +1,6 @@import { handleAndReturnErrorResponse } from "@/lib/api/errors";import { bulkDeletePartners } from "@/lib/api/partners/bulk-delete-partners";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { verifyQstashSignature } from "@/lib/cron/verify-qstash";import { prisma } from "@/lib/prisma";import { log } from "@dub/utils";@@ -32,7 +33,7 @@ export async function POST(req: Request) {none: {},},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (partnersToDelete.length === 0) {diff --git a/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.ts b/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.tsindex 440e120932..564517dd6f 100644--- a/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.ts+++ b/apps/web/app/(ee)/api/cron/partners/merge-accounts/route.ts@@ -4,6 +4,7 @@ import { linkCache } from "@/lib/api/links/cache";import { includeProgramEnrollment } from "@/lib/api/links/include-program-enrollment";import { includeTags } from "@/lib/api/links/include-tags";import { syncTotalCommissions } from "@/lib/api/partners/sync-total-commissions";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { verifyQstashSignature } from "@/lib/cron/verify-qstash";import { conn } from "@/lib/planetscale";import { prisma } from "@/lib/prisma";@@ -157,23 +158,35 @@ export async function POST(req: Request) {},};- // update links, commissions, bounty submissions, and payouts+ // update links and payoutsif (programIdsToTransfer.length > 0) {- const [- updatedLinksRes,- updatedCustomersRes,- updatedCommissionsRes,- updatedPayoutsRes,- ] = await Promise.all([+ const [updatedLinksRes, updatedPayoutsRes] = await Promise.all([prisma.link.updateMany(updateManyPayload),- prisma.customer.updateMany(updateManyPayload),- prisma.commission.updateMany(updateManyPayload),prisma.payout.updateMany(updateManyPayload),]);console.log(- `Updated ${updatedLinksRes.count} links, ${updatedCustomersRes.count} customers, ${updatedCommissionsRes.count} commissions, and ${updatedPayoutsRes.count} payouts`,+ `Updated ${updatedLinksRes.count} links, and ${updatedPayoutsRes.count} payouts`,);+ // for commissions / customers, we need to update them in batches of PRISMA_UPDATEMANY_LIMIT cause there can be a lot of them+ while (true) {+ const { count } = await prisma.commission.updateMany({+ ...updateManyPayload,+ limit: PRISMA_UPDATEMANY_LIMIT,+ });+ console.log(`Updated ${count} commissions`);+ if (count < PRISMA_UPDATEMANY_LIMIT) break;+ }++ while (true) {+ const { count } = await prisma.customer.updateMany({+ ...updateManyPayload,+ limit: PRISMA_UPDATEMANY_LIMIT,+ });+ console.log(`Updated ${count} customers`);+ if (count < PRISMA_UPDATEMANY_LIMIT) break;+ }+// update discount codes, notification emails, messages, and partner commentsconst [updatedDiscountCodesRes,diff --git a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.tsindex 72d245bc84..fd2a033497 100644--- a/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts+++ b/apps/web/app/(ee)/api/cron/webhooks/sync-click-workspaces/utils.ts@@ -1,3 +1,4 @@+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { prisma } from "@/lib/prisma";import { redis } from "@/lib/upstash";import { LINK_CLICKED_WEBHOOK_WORKSPACES_REDIS_KEY } from "@/lib/webhook/click-webhook-workspaces";@@ -64,7 +65,7 @@ export const cleanupRedundantLinkWebhookEntries = async () => {in: nonLinkScopeWebhooks.map((webhook) => webhook.id),},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});const deleted = await prisma.linkWebhook.deleteMany({where: {diff --git a/apps/web/lib/api/fraud/hold-pending-commissions.ts b/apps/web/lib/api/fraud/hold-pending-commissions.tsindex 7276ea2428..5ba5158db7 100644--- a/apps/web/lib/api/fraud/hold-pending-commissions.ts+++ b/apps/web/lib/api/fraud/hold-pending-commissions.ts@@ -1,4 +1,5 @@import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { prisma } from "@/lib/prisma";import { chunk, groupBy } from "@dub/utils";import { CommissionStatus, Prisma, ProgramEnrollment } from "@prisma/client";@@ -60,7 +61,7 @@ export async function holdPendingCommissions(},},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (pendingCommissions.length === 0) {diff --git a/apps/web/lib/api/fraud/hold-processed-commissions.ts b/apps/web/lib/api/fraud/hold-processed-commissions.tsindex 5e93e62492..fcb5962213 100644--- a/apps/web/lib/api/fraud/hold-processed-commissions.ts+++ b/apps/web/lib/api/fraud/hold-processed-commissions.ts@@ -1,4 +1,5 @@import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { retallyPayoutsAmount } from "@/lib/payouts/retally-payouts-amount";import { prisma } from "@/lib/prisma";import { chunk, groupBy } from "@dub/utils";@@ -72,7 +73,7 @@ export async function holdProcessedCommissions(},},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (processedCommissions.length === 0) {diff --git a/apps/web/lib/api/fraud/release-hold-commissions.ts b/apps/web/lib/api/fraud/release-hold-commissions.tsindex 959db0be41..c768b40e74 100644--- a/apps/web/lib/api/fraud/release-hold-commissions.ts+++ b/apps/web/lib/api/fraud/release-hold-commissions.ts@@ -2,7 +2,7 @@ import { triggerAggregateDueCommissionsCronJob } from "@/lib/actions/partners/trimport { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";import { syncTotalCommissions } from "@/lib/api/partners/sync-total-commissions";import { executeWorkflows } from "@/lib/api/workflows/execute-workflows";-import { qstash } from "@/lib/cron";+import { PRISMA_UPDATEMANY_LIMIT, qstash } from "@/lib/cron";import { prisma } from "@/lib/prisma";import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";import {@@ -129,7 +129,7 @@ export async function releaseHoldCommissions({earnings: true,status: true,},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (commissionsToRelease.length === 0) {diff --git a/apps/web/lib/cron/index.ts b/apps/web/lib/cron/index.tsindex 7b9eb8fdc8..dece18a598 100644--- a/apps/web/lib/cron/index.ts+++ b/apps/web/lib/cron/index.ts@@ -7,3 +7,6 @@ export const qstash = new Client({// Default batch size for cron jobs that process records in batchesexport const CRON_BATCH_SIZE = 100;++// 250 is generally the best size limit for Prisma updateMany operations+export const PRISMA_UPDATEMANY_LIMIT = 250;diff --git a/apps/web/scripts/customers/beehiiv/fix-case-a-complex.ts b/apps/web/scripts/customers/beehiiv/fix-case-a-complex.tsindex 294479a8b8..a29b3f485c 100644--- a/apps/web/scripts/customers/beehiiv/fix-case-a-complex.ts+++ b/apps/web/scripts/customers/beehiiv/fix-case-a-complex.ts@@ -1,4 +1,5 @@import { createId } from "@/lib/api/create-id";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { retallyPayoutsAmount } from "@/lib/payouts/retally-payouts-amount";import { prisma } from "@/lib/prisma";import { groupBy, linkConstructorSimple } from "@dub/utils";@@ -113,9 +114,9 @@ async function main() {data: {partnerId: link.actualPartnerId!,},- limit: 250,+ limit: PRISMA_UPDATEMANY_LIMIT,});- if (updatedCommissions.count < 250) {+ if (updatedCommissions.count < PRISMA_UPDATEMANY_LIMIT) {break;}console.log(diff --git a/apps/web/scripts/migrations/backfill-hold-pending-commissions.ts b/apps/web/scripts/migrations/backfill-hold-pending-commissions.tsindex 571e386fa3..093bf7614b 100644--- a/apps/web/scripts/migrations/backfill-hold-pending-commissions.ts+++ b/apps/web/scripts/migrations/backfill-hold-pending-commissions.ts@@ -2,6 +2,7 @@ import {CUSTOMER_LEVEL_FRAUD_RULES,PARTNER_LEVEL_FRAUD_RULES,} from "@/lib/api/fraud/constants";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { prisma } from "@/lib/prisma";import { groupBy } from "@dub/utils";import { CommissionStatus, FraudEventStatus, Prisma } from "@prisma/client";@@ -130,7 +131,7 @@ async function main() {},},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (commissions.length === 0) {diff --git a/apps/web/scripts/migrations/backfill-hold-processed-commissions.ts b/apps/web/scripts/migrations/backfill-hold-processed-commissions.tsindex 7d4578fe89..5d3d76639d 100644--- a/apps/web/scripts/migrations/backfill-hold-processed-commissions.ts+++ b/apps/web/scripts/migrations/backfill-hold-processed-commissions.ts@@ -2,6 +2,7 @@ import {CUSTOMER_LEVEL_FRAUD_RULES,PARTNER_LEVEL_FRAUD_RULES,} from "@/lib/api/fraud/constants";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { prisma } from "@/lib/prisma";import { groupBy } from "@dub/utils";import {@@ -142,7 +143,7 @@ async function main() {},},},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});if (commissions.length === 0) {diff --git a/apps/web/scripts/partners/update-payout-dates.ts b/apps/web/scripts/partners/update-payout-dates.tsindex a06b18fff9..e585d73efd 100644--- a/apps/web/scripts/partners/update-payout-dates.ts+++ b/apps/web/scripts/partners/update-payout-dates.ts@@ -1,3 +1,4 @@+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { prisma } from "@/lib/prisma";import "dotenv-flow/config";@@ -15,7 +16,7 @@ async function main() {orderBy: {createdAt: "desc",},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,});console.table(payoutsToUpdate);diff --git a/apps/web/scripts/programs/delete-program-enrollments.ts b/apps/web/scripts/programs/delete-program-enrollments.tsindex 34c35628ef..796021e233 100644--- a/apps/web/scripts/programs/delete-program-enrollments.ts+++ b/apps/web/scripts/programs/delete-program-enrollments.ts@@ -1,4 +1,5 @@import { bulkDeleteLinks } from "@/lib/api/links/bulk-delete-links";+import { PRISMA_UPDATEMANY_LIMIT } from "@/lib/cron";import { prisma } from "@/lib/prisma";import "dotenv-flow/config";@@ -11,7 +12,7 @@ async function main() {programId,totalLeads: 0,},- take: 250,+ take: PRISMA_UPDATEMANY_LIMIT,include: {links: true,},