Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Split aggregate-due-commissions into per-program workers with atomic claims.

Kiran K63359c2fbd6c
diff --git a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/process/route.ts b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/process/route.ts
new file mode 100644
index 0000000000..7d306546a0
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/process/route.ts
@@ -0,0 +1,349 @@
+import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";
+import { createId } from "@/lib/api/create-id";
+import { withCron } from "@/lib/cron/with-cron";
+import { prisma } from "@/lib/prisma";
+import { chunk } from "@dub/utils";
+import {
+ Commission,
+ CommissionStatus,
+ Payout,
+ PayoutStatus,
+ Prisma,
+ Program,
+} from "@prisma/client";
+import * as z from "zod/v4";
+import { logAndRespond } from "../../../utils";
+
+export const maxDuration = 600;
+export const dynamic = "force-dynamic";
+
+const BATCH_SIZE = 100;
+
+const inputSchema = z.object({
+ programId: z.string(),
+});
+
+// POST /api/cron/payouts/aggregate-due-commissions/process
+export const POST = withCron(async ({ rawBody }) => {
+ const { programId } = inputSchema.parse(JSON.parse(rawBody));
+
+ const program = await prisma.program.findUnique({
+ where: {
+ id: programId,
+ },
+ select: {
+ id: true,
+ name: true,
+ workspaceId: true,
+ },
+ });
+
+ if (!program) {
+ return logAndRespond(`Program ${programId} not found. Skipping...`);
+ }
+
+ const partnerGroupsByHoldingPeriod = await prisma.partnerGroup.groupBy({
+ by: ["holdingPeriodDays"],
+ where: {
+ programId,
+ },
+ _count: {
+ id: true,
+ },
+ orderBy: {
+ _count: {
+ id: "desc",
+ },
+ },
+ });
+
+ for (const { holdingPeriodDays } of partnerGroupsByHoldingPeriod) {
+ const partnerGroups = await prisma.partnerGroup.findMany({
+ where: {
+ holdingPeriodDays,
+ programId,
+ },
+ select: {
+ id: true,
+ },
+ });
+
+ console.log(
+ `Found ${partnerGroups.length} partner groups with holding period days: ${holdingPeriodDays}`,
+ );
+
+ while (true) {
+ const dueCommissions = await prisma.commission.findMany({
+ where: {
+ programId,
+ status: CommissionStatus.pending,
+ programEnrollment: {
+ groupId: {
+ in: partnerGroups.map((p) => p.id),
+ },
+ },
+ // If holding period days is greater than 0:
+ // we only process commissions that were created before the holding period
+ // but custom commissions are always included
+ ...(holdingPeriodDays > 0
+ ? {
+ OR: [
+ {
+ type: {
+ // includes manual + referral commissions + clawbacks
+ in: ["custom", "referral"],
+ },
+ },
+ {
+ createdAt: {
+ lt: new Date(
+ Date.now() - holdingPeriodDays * 24 * 60 * 60 * 1000,
+ ),
+ },
+ },
+ ],
+ }
+ : {}),
+ },
+ select: {
+ id: true,
+ createdAt: true,
+ amount: true,
+ earnings: true,
+ status: true,
+ partnerId: true,
+ programId: true,
+ },
+ orderBy: {
+ createdAt: "asc",
+ },
+ take: BATCH_SIZE,
+ });
+
+ if (dueCommissions.length === 0) {
+ console.log(
+ `No more due commissions found for partner groups with holding period days: ${holdingPeriodDays}. Skipping...`,
+ );
+ break;
+ }
+
+ console.log(
+ `Found ${dueCommissions.length} due commissions for partner groups with holding period days: ${holdingPeriodDays}`,
+ );
+
+ const commissionsByPartner = dueCommissions.reduce<
+ Record<string, typeof dueCommissions>
+ >((acc, commission) => {
+ if (!acc[commission.partnerId]) {
+ acc[commission.partnerId] = [];
+ }
+ acc[commission.partnerId].push(commission);
+ return acc;
+ }, {});
+
+ const partnersWithCommissions = Object.entries(commissionsByPartner).map(
+ ([partnerId, commissions]) => ({ partnerId, commissions }),
+ );
+
+ let totalProcessed = 0;
+ const partnerBatches = chunk(partnersWithCommissions, 50);
+
+ for (let i = 0; i < partnerBatches.length; i++) {
+ const partnerChunk = partnerBatches[i];
+
+ const existingPendingPayouts = await prisma.payout.findMany({
+ where: {
+ programId,
+ partnerId: {
+ in: partnerChunk.map(({ partnerId }) => partnerId),
+ },
+ status: PayoutStatus.pending,
+ },
+ select: {
+ id: true,
+ partnerId: true,
+ },
+ });
+
+ const pendingPayoutByPartnerId = new Map(
+ existingPendingPayouts.map((payout) => [payout.partnerId, payout]),
+ );
+
+ const results = await Promise.allSettled(
+ partnerChunk.map(({ partnerId, commissions }) =>
+ aggregateDueCommissionsForPartner({
+ partnerId,
+ program,
+ commissions,
+ existingPendingPayout: pendingPayoutByPartnerId.get(partnerId),
+ }),
+ ),
+ );
+
+ for (const result of results) {
+ if (result.status === "fulfilled" && result.value) {
+ totalProcessed++;
+ } else if (result.status === "rejected") {
+ console.error(
+ `Failed to aggregate due commissions for partner:`,
+ result.reason,
+ );
+ }
+ }
+
+ console.log(`Processed chunk ${i + 1} of ${partnerBatches.length}`);
+ }
+
+ const successRate =
+ (totalProcessed / partnersWithCommissions.length) * 100;
+
+ console.log(
+ `Processed ${totalProcessed}/${partnersWithCommissions.length} partners with due commissions for partner groups with holding period days: ${holdingPeriodDays} (${successRate.toFixed(1)}% success rate)`,
+ );
+
+ if (totalProcessed === 0) {
+ console.log(
+ `No partners were processed for holding period days: ${holdingPeriodDays}.`,
+ );
+ break;
+ }
+ }
+ }
+
+ return logAndRespond(
+ "Finished aggregating due commissions into payouts for all batches.",
+ );
+});
+
+async function aggregateDueCommissionsForPartner({
+ partnerId,
+ program,
+ commissions,
+ existingPendingPayout,
+}: {
+ partnerId: string;
+ program: Pick<Program, "id" | "name" | "workspaceId">;
+ commissions: Pick<
+ Commission,
+ "id" | "createdAt" | "amount" | "earnings" | "status"
+ >[];
+ existingPendingPayout?: Pick<Payout, "id" | "partnerId">;
+}): Promise<boolean> {
+ // Sort the commissions by createdAt
+ const sortedCommissions = commissions.sort(
+ (a, b) => a.createdAt.getTime() - b.createdAt.getTime(),
+ );
+
+ const commissionIds = sortedCommissions.map((c) => c.id);
+ const periodStart = sortedCommissions[0].createdAt;
+ const periodEnd = sortedCommissions[sortedCommissions.length - 1].createdAt;
+
+ let payoutToUse = existingPendingPayout
+ ? { id: existingPendingPayout.id }
+ : null;
+
+ const isReusingPendingPayout = !!payoutToUse;
+
+ if (!payoutToUse) {
+ // Amount starts at 0 and is set from actually-claimed commissions below
+ payoutToUse = await prisma.payout.create({
+ data: {
+ id: createId({ prefix: "po_" }),
+ programId: program.id,
+ partnerId,
+ periodStart,
+ periodEnd,
+ amount: 0,
+ description: `Dub Partners payout (${program.name})`,
+ },
+ select: {
+ id: true,
+ },
+ });
+ }
+
+ // Use raw SQL instead of prisma.commission.updateMany.
+ // Prisma has a reported MySQL issue where updateMany may drop WHERE predicates
+ // during the UPDATE, allowing concurrent workers to claim the same commissions.
+ // See: https://github.com/prisma/prisma/issues/28840
+ const updatedCommissions = await prisma.$executeRaw`
+ UPDATE Commission
+ SET
+ status = ${CommissionStatus.processed},
+ payoutId = ${payoutToUse.id},
+ updatedAt = NOW()
+ WHERE id IN (${Prisma.join(commissionIds)})
+ AND programId = ${program.id}
+ AND partnerId = ${partnerId}
+ AND status = ${CommissionStatus.pending}
+ `;
+
+ if (updatedCommissions === 0) {
+ console.warn(
+ `No commissions were updated for partner ${partnerId}. Skipping...`,
+ );
+
+ // Another worker won the race — only delete if still empty.
+ if (!isReusingPendingPayout) {
+ await prisma.payout.deleteMany({
+ where: {
+ id: payoutToUse.id,
+ commissions: {
+ none: {},
+ },
+ },
+ });
+ }
+
+ return false;
+ }
+
+ // Always set amount from DB after claim (create + reuse) so partial claims
+ // and concurrent attaches cannot leave a stale precomputed sum.
+ const {
+ _sum: { earnings: totalEarningsForPayout },
+ } = await prisma.commission.aggregate({
+ where: {
+ payoutId: payoutToUse.id,
+ },
+ _sum: {
+ earnings: true,
+ },
+ });
+
+ await prisma.payout.update({
+ where: {
+ id: payoutToUse.id,
+ },
+ data: {
+ amount: totalEarningsForPayout ?? 0,
+ ...(isReusingPendingPayout ? { periodEnd } : {}),
+ },
+ });
+
+ // Only activity-log commissions we actually claimed (handles partial races).
+ const claimedCommissions = await prisma.commission.findMany({
+ where: {
+ id: {
+ in: commissionIds,
+ },
+ payoutId: payoutToUse.id,
+ status: CommissionStatus.processed,
+ },
+ select: {
+ id: true,
+ amount: true,
+ earnings: true,
+ status: true,
+ },
+ });
+
+ await trackCommissionStatusUpdate({
+ workspaceId: program.workspaceId,
+ programId: program.id,
+ commissions: claimedCommissions,
+ newStatus: CommissionStatus.processed,
+ });
+
+ return true;
+}
diff --git a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts
index 80299c22a4..8418f66609 100644
--- a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts
+++ b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts
@@ -1,276 +1,59 @@
-import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";
-import { createId } from "@/lib/api/create-id";
import { qstash } from "@/lib/cron";
import { withCron } from "@/lib/cron/with-cron";
import { prisma } from "@/lib/prisma";
import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";
-import * as z from "zod/v4";
+import { CommissionStatus } from "@prisma/client";
+import { PublishBatchRequest } from "@upstash/qstash";
import { logAndRespond } from "../../utils";
+export const maxDuration = 600;
export const dynamic = "force-dynamic";
-const BATCH_SIZE = 1000;
-
-const schema = z.object({
- programId: z.string().optional().describe("Optional program ID to filter by"),
-});
-
// This cron job aggregates due commissions (pending commissions that are past the partner group's holding period) into payouts.
// Runs once every hour (0 * * * *) + calls itself recursively to look through all pending commissions available.
-// GET/POST /api/cron/payouts/aggregate-due-commissions
+// GET /api/cron/payouts/aggregate-due-commissions
export const GET = withCron(async ({ rawBody }) => {
- let { programId } = schema.parse(rawBody ? JSON.parse(rawBody) : {});
-
- const partnerGroupsByHoldingPeriod = await prisma.partnerGroup.groupBy({
- by: ["holdingPeriodDays"],
- ...(programId ? { where: { programId } } : {}),
+ const programsWithDueCommissions = await prisma.commission.groupBy({
+ by: ["programId"],
+ where: {
+ status: CommissionStatus.pending,
+ },
_count: {
id: true,
},
- orderBy: {
- _count: {
- id: "desc",
- },
- },
});
- console.log(JSON.stringify(partnerGroupsByHoldingPeriod, null, 2));
-
- let holdingPeriodsWithMoreToProcess: number[] = [];
- for (const { holdingPeriodDays } of partnerGroupsByHoldingPeriod) {
- const partnerGroups = await prisma.partnerGroup.findMany({
- where: {
- holdingPeriodDays,
- ...(programId ? { programId } : {}),
- },
- select: {
- id: true,
- program: {
- select: {
- id: true,
- name: true,
- workspaceId: true,
- },
- },
- },
- });
-
- console.log(
- `Found ${partnerGroups.length} partner groups with holding period days: ${holdingPeriodDays}`,
- );
-
- // Find all due commissions (limit by BATCH_SIZE)
- const dueCommissions = await prisma.commission.findMany({
- where: {
- status: "pending",
- programEnrollment: {
- groupId: {
- in: partnerGroups.map((p) => p.id),
- },
- },
- // If holding period days is greater than 0:
- // we only process commissions that were created before the holding period
- // but custom commissions are always included
- ...(holdingPeriodDays > 0
- ? {
- OR: [
- {
- type: {
- // includes manual + referral commissions + clawbacks
- in: ["custom", "referral"],
- },
- },
- {
- createdAt: {
- lt: new Date(
- Date.now() - holdingPeriodDays * 24 * 60 * 60 * 1000,
- ),
- },
- },
- ],
- }
- : {}),
- },
- select: {
- id: true,
- createdAt: true,
- amount: true,
- earnings: true,
- status: true,
- partnerId: true,
- programId: true,
- },
- orderBy: {
- createdAt: "asc",
- },
- take: BATCH_SIZE,
- });
-
- if (dueCommissions.length === 0) {
- console.log(
- `No more due commissions found for partner groups with holding period days: ${holdingPeriodDays}, skipping...`,
- );
- continue;
- }
-
- if (dueCommissions.length === BATCH_SIZE) {
- holdingPeriodsWithMoreToProcess.push(holdingPeriodDays);
- }
+ const programIds = programsWithDueCommissions.map((p) => p.programId);
- console.log(
- `Found ${dueCommissions.length} due commissions for partner groups with holding period days: ${holdingPeriodDays}`,
- );
+ if (programIds.length === 0) {
+ return logAndRespond("No programs with due commissions found. Skipping...");
+ }
- const partnerProgramCommissions = dueCommissions.reduce<
- Record<string, typeof dueCommissions>
- >((acc, commission) => {
- const key = `${commission.partnerId}:${commission.programId}`;
- if (!acc[key]) {
- acc[key] = [];
- }
- acc[key].push(commission);
- return acc;
- }, {});
+ console.table(programsWithDueCommissions);
- const partnerProgramCommissionsArray = Object.entries(
- partnerProgramCommissions,
- ).map(([key, commissions]) => ({
- partnerId: key.split(":")[0],
- programId: key.split(":")[1],
- commissions,
- }));
+ const programIdChunks = chunk(programIds, 50);
- const existingPendingPayouts = await prisma.payout.findMany({
- where: {
- programId: {
- in: partnerProgramCommissionsArray.map((p) => p.programId),
+ for (const [index, programIdChunk] of programIdChunks.entries()) {
+ const jobs: PublishBatchRequest<{ programId: string }>[] =
+ programIdChunk.map((programId) => ({
+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions/process`,
+ body: {
+ programId,
},
- partnerId: {
- in: partnerProgramCommissionsArray.map((p) => p.partnerId),
+ deduplicationId: `aggregate-due-commissions-${programId}`,
+ label: "aggregate-due-commissions",
+ flowControl: {
+ key: `aggregate-due-commissions-${programId}`,
+ parallelism: 1,
+ rate: 5,
+ period: "1s",
},
- status: "pending",
- },
- });
-
- console.log(
- `Processing ${partnerProgramCommissionsArray.length} partners with due commissions for partner groups with holding period days: ${holdingPeriodDays}`,
- );
- let totalProcessed = 0;
-
- const chunks = chunk(partnerProgramCommissionsArray, 50);
- for (let i = 0; i < chunks.length; i++) {
- const chunk = chunks[i];
- await Promise.allSettled(
- chunk.map(async ({ partnerId, programId, commissions }) => {
- // sort the commissions by createdAt
- const sortedCommissions = commissions.sort(
- (a, b) => a.createdAt.getTime() - b.createdAt.getTime(),
- );
-
- // sum the earnings of the commissions
- const totalEarnings = sortedCommissions.reduce(
- (total, commission) => total + commission.earnings,
- 0,
- );
-
- // earliest commission date
- const periodStart = sortedCommissions[0].createdAt;
-
- // last commission date
- const periodEnd =
- sortedCommissions[sortedCommissions.length - 1].createdAt;
-
- let payoutToUse = existingPendingPayouts.find(
- (p) => p.partnerId === partnerId && p.programId === programId,
- );
-
- if (!payoutToUse) {
- const programName = partnerGroups.find(
- (p) => p.program.id === programId,
- )?.program.name;
- payoutToUse = await prisma.payout.create({
- data: {
- id: createId({ prefix: "po_" }),
- programId,
- partnerId,
- periodStart,
- periodEnd,
- amount: totalEarnings,
- description: `Dub Partners payout${programName ? ` (${programName})` : ""}`,
- },
- });
- }
-
- // update the commissions to have the payoutId
- await prisma.commission.updateMany({
- where: {
- id: { in: commissions.map((c) => c.id) },
- },
- data: {
- status: "processed",
- payoutId: payoutToUse.id,
- },
- });
-
- const workspaceId = partnerGroups.find(
- (p) => p.program.id === programId,
- )?.program.workspaceId;
-
- if (workspaceId) {
- await trackCommissionStatusUpdate({
- workspaceId,
- programId,
- commissions,
- newStatus: "processed",
- });
- }
+ }));
- // if we're reusing a pending payout, we need to update the amount and periodEnd
- if (existingPendingPayouts.find((p) => p.id === payoutToUse.id)) {
- await prisma.payout.update({
- where: {
- id: payoutToUse.id,
- },
- data: {
- amount: {
- increment: totalEarnings,
- },
- periodEnd,
- },
- });
- }
+ await qstash.batchJSON(jobs);
- totalProcessed++;
- }),
- );
- console.log(`Processed chunk ${i + 1} of ${chunks.length}`);
- }
-
- const successRate =
- (totalProcessed / partnerProgramCommissionsArray.length) * 100;
- console.log(
- `Processed ${totalProcessed}/${partnerProgramCommissionsArray.length} partners with due commissions for partner groups with holding period days: ${holdingPeriodDays} (${successRate.toFixed(1)}% success rate)`,
- );
- }
-
- if (holdingPeriodsWithMoreToProcess.length > 0) {
console.log(
- `Several holding periods still have more due commissions: ${holdingPeriodsWithMoreToProcess.join(", ")}`,
- );
-
- const qstashResponse = await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`,
- body: programId ? { programId } : {}, // pass programId if defined, else pass an empty object
- });
-
- if (!qstashResponse.messageId) {
- throw new Error(
- `Error sending message to Qstash to schedule next batch of payouts: ${JSON.stringify(qstashResponse)}`,
- );
- }
-
- return logAndRespond(
- "Finished aggregating due commissions into payouts for current batch. Scheduling next batch...",
+ `Enqueued index ${index + 1}/${programIdChunks.length} jobs to be processed.`,
);
}
@@ -278,5 +61,3 @@ export const GET = withCron(async ({ rawBody }) => {
"Finished aggregating due commissions into payouts for all batches.",
);
});
-
-export const POST = GET;
diff --git a/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.ts b/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.ts
index eea3c1e018..62542a1ed7 100644
--- a/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.ts
+++ b/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.ts
@@ -3,9 +3,10 @@ import { APP_DOMAIN_WITH_NGROK, prettyPrint } from "@dub/utils";
export async function triggerAggregateDueCommissionsCronJob(programId: string) {
const qstashResponse = await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`,
+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions/process`,
+ label: "aggregate-due-commissions",
flowControl: {
- key: programId,
+ key: `aggregate-due-commissions-${programId}`,
parallelism: 1,
},
body: {
diff --git a/apps/web/scripts/programs/backfill-reuse-commission.ts b/apps/web/scripts/programs/backfill-reuse-commission.ts
index cc4091c768..4ce58e1bd2 100644
--- a/apps/web/scripts/programs/backfill-reuse-commission.ts
+++ b/apps/web/scripts/programs/backfill-reuse-commission.ts
@@ -1,9 +1,9 @@
+import { triggerAggregateDueCommissionsCronJob } from "@/lib/actions/partners/trigger-aggregate-due-commissions";
import { isFirstConversion } from "@/lib/analytics/is-first-conversion";
import { createId } from "@/lib/api/create-id";
import { updateLinkStatsForImporter } from "@/lib/api/links/update-link-stats-for-importer";
import { syncPartnerLinksStats } from "@/lib/api/partners/sync-partner-links-stats";
import { executeWorkflows } from "@/lib/api/workflows/execute-workflows";
-import { qstash } from "@/lib/cron";
import { queuePartnerCommissionCreation } from "@/lib/partners/queue-partner-commission-creation";
import { prisma } from "@/lib/prisma";
import { getCustomerEventsTB } from "@/lib/tinybird/get-customer-events-tb";
@@ -16,7 +16,7 @@ import { recordSaleWithTimestamp } from "@/lib/tinybird/record-sale";
import { CreatePartnerCommissionProps } from "@/lib/types";
import { leadEventSchemaTB } from "@/lib/zod/schemas/leads";
import { saleEventSchemaTB } from "@/lib/zod/schemas/sales";
-import { APP_DOMAIN_WITH_NGROK, nanoid, prettyPrint } from "@dub/utils";
+import { nanoid } from "@dub/utils";
import "dotenv-flow/config";
import * as z from "zod/v4";
@@ -343,13 +343,7 @@ async function main() {
]);
}
- const qstashResponse = await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`,
- body: { programId },
- });
- console.log(
- `Triggered aggregate due commissions cron job for program ${programId}: ${prettyPrint(qstashResponse)}`,
- );
+ await triggerAggregateDueCommissionsCronJob(programId);
}
main();