The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Split aggregate-due-commissions into per-program workers with atomic claims.
Kiran K63359c2fbd6c
diff --git a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/process/route.ts b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/process/route.tsnew file mode 100644index 0000000000..7d306546a0--- /dev/null+++ b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/process/route.ts@@ -0,0 +1,349 @@+import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";+import { createId } from "@/lib/api/create-id";+import { withCron } from "@/lib/cron/with-cron";+import { prisma } from "@/lib/prisma";+import { chunk } from "@dub/utils";+import {+ Commission,+ CommissionStatus,+ Payout,+ PayoutStatus,+ Prisma,+ Program,+} from "@prisma/client";+import * as z from "zod/v4";+import { logAndRespond } from "../../../utils";++export const maxDuration = 600;+export const dynamic = "force-dynamic";++const BATCH_SIZE = 100;++const inputSchema = z.object({+ programId: z.string(),+});++// POST /api/cron/payouts/aggregate-due-commissions/process+export const POST = withCron(async ({ rawBody }) => {+ const { programId } = inputSchema.parse(JSON.parse(rawBody));++ const program = await prisma.program.findUnique({+ where: {+ id: programId,+ },+ select: {+ id: true,+ name: true,+ workspaceId: true,+ },+ });++ if (!program) {+ return logAndRespond(`Program ${programId} not found. Skipping...`);+ }++ const partnerGroupsByHoldingPeriod = await prisma.partnerGroup.groupBy({+ by: ["holdingPeriodDays"],+ where: {+ programId,+ },+ _count: {+ id: true,+ },+ orderBy: {+ _count: {+ id: "desc",+ },+ },+ });++ for (const { holdingPeriodDays } of partnerGroupsByHoldingPeriod) {+ const partnerGroups = await prisma.partnerGroup.findMany({+ where: {+ holdingPeriodDays,+ programId,+ },+ select: {+ id: true,+ },+ });++ console.log(+ `Found ${partnerGroups.length} partner groups with holding period days: ${holdingPeriodDays}`,+ );++ while (true) {+ const dueCommissions = await prisma.commission.findMany({+ where: {+ programId,+ status: CommissionStatus.pending,+ programEnrollment: {+ groupId: {+ in: partnerGroups.map((p) => p.id),+ },+ },+ // If holding period days is greater than 0:+ // we only process commissions that were created before the holding period+ // but custom commissions are always included+ ...(holdingPeriodDays > 0+ ? {+ OR: [+ {+ type: {+ // includes manual + referral commissions + clawbacks+ in: ["custom", "referral"],+ },+ },+ {+ createdAt: {+ lt: new Date(+ Date.now() - holdingPeriodDays * 24 * 60 * 60 * 1000,+ ),+ },+ },+ ],+ }+ : {}),+ },+ select: {+ id: true,+ createdAt: true,+ amount: true,+ earnings: true,+ status: true,+ partnerId: true,+ programId: true,+ },+ orderBy: {+ createdAt: "asc",+ },+ take: BATCH_SIZE,+ });++ if (dueCommissions.length === 0) {+ console.log(+ `No more due commissions found for partner groups with holding period days: ${holdingPeriodDays}. Skipping...`,+ );+ break;+ }++ console.log(+ `Found ${dueCommissions.length} due commissions for partner groups with holding period days: ${holdingPeriodDays}`,+ );++ const commissionsByPartner = dueCommissions.reduce<+ Record<string, typeof dueCommissions>+ >((acc, commission) => {+ if (!acc[commission.partnerId]) {+ acc[commission.partnerId] = [];+ }+ acc[commission.partnerId].push(commission);+ return acc;+ }, {});++ const partnersWithCommissions = Object.entries(commissionsByPartner).map(+ ([partnerId, commissions]) => ({ partnerId, commissions }),+ );++ let totalProcessed = 0;+ const partnerBatches = chunk(partnersWithCommissions, 50);++ for (let i = 0; i < partnerBatches.length; i++) {+ const partnerChunk = partnerBatches[i];++ const existingPendingPayouts = await prisma.payout.findMany({+ where: {+ programId,+ partnerId: {+ in: partnerChunk.map(({ partnerId }) => partnerId),+ },+ status: PayoutStatus.pending,+ },+ select: {+ id: true,+ partnerId: true,+ },+ });++ const pendingPayoutByPartnerId = new Map(+ existingPendingPayouts.map((payout) => [payout.partnerId, payout]),+ );++ const results = await Promise.allSettled(+ partnerChunk.map(({ partnerId, commissions }) =>+ aggregateDueCommissionsForPartner({+ partnerId,+ program,+ commissions,+ existingPendingPayout: pendingPayoutByPartnerId.get(partnerId),+ }),+ ),+ );++ for (const result of results) {+ if (result.status === "fulfilled" && result.value) {+ totalProcessed++;+ } else if (result.status === "rejected") {+ console.error(+ `Failed to aggregate due commissions for partner:`,+ result.reason,+ );+ }+ }++ console.log(`Processed chunk ${i + 1} of ${partnerBatches.length}`);+ }++ const successRate =+ (totalProcessed / partnersWithCommissions.length) * 100;++ console.log(+ `Processed ${totalProcessed}/${partnersWithCommissions.length} partners with due commissions for partner groups with holding period days: ${holdingPeriodDays} (${successRate.toFixed(1)}% success rate)`,+ );++ if (totalProcessed === 0) {+ console.log(+ `No partners were processed for holding period days: ${holdingPeriodDays}.`,+ );+ break;+ }+ }+ }++ return logAndRespond(+ "Finished aggregating due commissions into payouts for all batches.",+ );+});++async function aggregateDueCommissionsForPartner({+ partnerId,+ program,+ commissions,+ existingPendingPayout,+}: {+ partnerId: string;+ program: Pick<Program, "id" | "name" | "workspaceId">;+ commissions: Pick<+ Commission,+ "id" | "createdAt" | "amount" | "earnings" | "status"+ >[];+ existingPendingPayout?: Pick<Payout, "id" | "partnerId">;+}): Promise<boolean> {+ // Sort the commissions by createdAt+ const sortedCommissions = commissions.sort(+ (a, b) => a.createdAt.getTime() - b.createdAt.getTime(),+ );++ const commissionIds = sortedCommissions.map((c) => c.id);+ const periodStart = sortedCommissions[0].createdAt;+ const periodEnd = sortedCommissions[sortedCommissions.length - 1].createdAt;++ let payoutToUse = existingPendingPayout+ ? { id: existingPendingPayout.id }+ : null;++ const isReusingPendingPayout = !!payoutToUse;++ if (!payoutToUse) {+ // Amount starts at 0 and is set from actually-claimed commissions below+ payoutToUse = await prisma.payout.create({+ data: {+ id: createId({ prefix: "po_" }),+ programId: program.id,+ partnerId,+ periodStart,+ periodEnd,+ amount: 0,+ description: `Dub Partners payout (${program.name})`,+ },+ select: {+ id: true,+ },+ });+ }++ // Use raw SQL instead of prisma.commission.updateMany.+ // Prisma has a reported MySQL issue where updateMany may drop WHERE predicates+ // during the UPDATE, allowing concurrent workers to claim the same commissions.+ // See: https://github.com/prisma/prisma/issues/28840+ const updatedCommissions = await prisma.$executeRaw`+ UPDATE Commission+ SET+ status = ${CommissionStatus.processed},+ payoutId = ${payoutToUse.id},+ updatedAt = NOW()+ WHERE id IN (${Prisma.join(commissionIds)})+ AND programId = ${program.id}+ AND partnerId = ${partnerId}+ AND status = ${CommissionStatus.pending}+ `;++ if (updatedCommissions === 0) {+ console.warn(+ `No commissions were updated for partner ${partnerId}. Skipping...`,+ );++ // Another worker won the race — only delete if still empty.+ if (!isReusingPendingPayout) {+ await prisma.payout.deleteMany({+ where: {+ id: payoutToUse.id,+ commissions: {+ none: {},+ },+ },+ });+ }++ return false;+ }++ // Always set amount from DB after claim (create + reuse) so partial claims+ // and concurrent attaches cannot leave a stale precomputed sum.+ const {+ _sum: { earnings: totalEarningsForPayout },+ } = await prisma.commission.aggregate({+ where: {+ payoutId: payoutToUse.id,+ },+ _sum: {+ earnings: true,+ },+ });++ await prisma.payout.update({+ where: {+ id: payoutToUse.id,+ },+ data: {+ amount: totalEarningsForPayout ?? 0,+ ...(isReusingPendingPayout ? { periodEnd } : {}),+ },+ });++ // Only activity-log commissions we actually claimed (handles partial races).+ const claimedCommissions = await prisma.commission.findMany({+ where: {+ id: {+ in: commissionIds,+ },+ payoutId: payoutToUse.id,+ status: CommissionStatus.processed,+ },+ select: {+ id: true,+ amount: true,+ earnings: true,+ status: true,+ },+ });++ await trackCommissionStatusUpdate({+ workspaceId: program.workspaceId,+ programId: program.id,+ commissions: claimedCommissions,+ newStatus: CommissionStatus.processed,+ });++ return true;+}diff --git a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.tsindex 80299c22a4..8418f66609 100644--- a/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts+++ b/apps/web/app/(ee)/api/cron/payouts/aggregate-due-commissions/route.ts@@ -1,276 +1,59 @@-import { trackCommissionStatusUpdate } from "@/lib/api/commissions/track-commission-update-activity-log";-import { createId } from "@/lib/api/create-id";import { qstash } from "@/lib/cron";import { withCron } from "@/lib/cron/with-cron";import { prisma } from "@/lib/prisma";import { APP_DOMAIN_WITH_NGROK, chunk } from "@dub/utils";-import * as z from "zod/v4";+import { CommissionStatus } from "@prisma/client";+import { PublishBatchRequest } from "@upstash/qstash";import { logAndRespond } from "../../utils";+export const maxDuration = 600;export const dynamic = "force-dynamic";-const BATCH_SIZE = 1000;--const schema = z.object({- programId: z.string().optional().describe("Optional program ID to filter by"),-});-// This cron job aggregates due commissions (pending commissions that are past the partner group's holding period) into payouts.// Runs once every hour (0 * * * *) + calls itself recursively to look through all pending commissions available.-// GET/POST /api/cron/payouts/aggregate-due-commissions+// GET /api/cron/payouts/aggregate-due-commissionsexport const GET = withCron(async ({ rawBody }) => {- let { programId } = schema.parse(rawBody ? JSON.parse(rawBody) : {});-- const partnerGroupsByHoldingPeriod = await prisma.partnerGroup.groupBy({- by: ["holdingPeriodDays"],- ...(programId ? { where: { programId } } : {}),+ const programsWithDueCommissions = await prisma.commission.groupBy({+ by: ["programId"],+ where: {+ status: CommissionStatus.pending,+ },_count: {id: true,},- orderBy: {- _count: {- id: "desc",- },- },});- console.log(JSON.stringify(partnerGroupsByHoldingPeriod, null, 2));-- let holdingPeriodsWithMoreToProcess: number[] = [];- for (const { holdingPeriodDays } of partnerGroupsByHoldingPeriod) {- const partnerGroups = await prisma.partnerGroup.findMany({- where: {- holdingPeriodDays,- ...(programId ? { programId } : {}),- },- select: {- id: true,- program: {- select: {- id: true,- name: true,- workspaceId: true,- },- },- },- });-- console.log(- `Found ${partnerGroups.length} partner groups with holding period days: ${holdingPeriodDays}`,- );-- // Find all due commissions (limit by BATCH_SIZE)- const dueCommissions = await prisma.commission.findMany({- where: {- status: "pending",- programEnrollment: {- groupId: {- in: partnerGroups.map((p) => p.id),- },- },- // If holding period days is greater than 0:- // we only process commissions that were created before the holding period- // but custom commissions are always included- ...(holdingPeriodDays > 0- ? {- OR: [- {- type: {- // includes manual + referral commissions + clawbacks- in: ["custom", "referral"],- },- },- {- createdAt: {- lt: new Date(- Date.now() - holdingPeriodDays * 24 * 60 * 60 * 1000,- ),- },- },- ],- }- : {}),- },- select: {- id: true,- createdAt: true,- amount: true,- earnings: true,- status: true,- partnerId: true,- programId: true,- },- orderBy: {- createdAt: "asc",- },- take: BATCH_SIZE,- });-- if (dueCommissions.length === 0) {- console.log(- `No more due commissions found for partner groups with holding period days: ${holdingPeriodDays}, skipping...`,- );- continue;- }-- if (dueCommissions.length === BATCH_SIZE) {- holdingPeriodsWithMoreToProcess.push(holdingPeriodDays);- }+ const programIds = programsWithDueCommissions.map((p) => p.programId);- console.log(- `Found ${dueCommissions.length} due commissions for partner groups with holding period days: ${holdingPeriodDays}`,- );+ if (programIds.length === 0) {+ return logAndRespond("No programs with due commissions found. Skipping...");+ }- const partnerProgramCommissions = dueCommissions.reduce<- Record<string, typeof dueCommissions>- >((acc, commission) => {- const key = `${commission.partnerId}:${commission.programId}`;- if (!acc[key]) {- acc[key] = [];- }- acc[key].push(commission);- return acc;- }, {});+ console.table(programsWithDueCommissions);- const partnerProgramCommissionsArray = Object.entries(- partnerProgramCommissions,- ).map(([key, commissions]) => ({- partnerId: key.split(":")[0],- programId: key.split(":")[1],- commissions,- }));+ const programIdChunks = chunk(programIds, 50);- const existingPendingPayouts = await prisma.payout.findMany({- where: {- programId: {- in: partnerProgramCommissionsArray.map((p) => p.programId),+ for (const [index, programIdChunk] of programIdChunks.entries()) {+ const jobs: PublishBatchRequest<{ programId: string }>[] =+ programIdChunk.map((programId) => ({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions/process`,+ body: {+ programId,},- partnerId: {- in: partnerProgramCommissionsArray.map((p) => p.partnerId),+ deduplicationId: `aggregate-due-commissions-${programId}`,+ label: "aggregate-due-commissions",+ flowControl: {+ key: `aggregate-due-commissions-${programId}`,+ parallelism: 1,+ rate: 5,+ period: "1s",},- status: "pending",- },- });-- console.log(- `Processing ${partnerProgramCommissionsArray.length} partners with due commissions for partner groups with holding period days: ${holdingPeriodDays}`,- );- let totalProcessed = 0;-- const chunks = chunk(partnerProgramCommissionsArray, 50);- for (let i = 0; i < chunks.length; i++) {- const chunk = chunks[i];- await Promise.allSettled(- chunk.map(async ({ partnerId, programId, commissions }) => {- // sort the commissions by createdAt- const sortedCommissions = commissions.sort(- (a, b) => a.createdAt.getTime() - b.createdAt.getTime(),- );-- // sum the earnings of the commissions- const totalEarnings = sortedCommissions.reduce(- (total, commission) => total + commission.earnings,- 0,- );-- // earliest commission date- const periodStart = sortedCommissions[0].createdAt;-- // last commission date- const periodEnd =- sortedCommissions[sortedCommissions.length - 1].createdAt;-- let payoutToUse = existingPendingPayouts.find(- (p) => p.partnerId === partnerId && p.programId === programId,- );-- if (!payoutToUse) {- const programName = partnerGroups.find(- (p) => p.program.id === programId,- )?.program.name;- payoutToUse = await prisma.payout.create({- data: {- id: createId({ prefix: "po_" }),- programId,- partnerId,- periodStart,- periodEnd,- amount: totalEarnings,- description: `Dub Partners payout${programName ? ` (${programName})` : ""}`,- },- });- }-- // update the commissions to have the payoutId- await prisma.commission.updateMany({- where: {- id: { in: commissions.map((c) => c.id) },- },- data: {- status: "processed",- payoutId: payoutToUse.id,- },- });-- const workspaceId = partnerGroups.find(- (p) => p.program.id === programId,- )?.program.workspaceId;-- if (workspaceId) {- await trackCommissionStatusUpdate({- workspaceId,- programId,- commissions,- newStatus: "processed",- });- }+ }));- // if we're reusing a pending payout, we need to update the amount and periodEnd- if (existingPendingPayouts.find((p) => p.id === payoutToUse.id)) {- await prisma.payout.update({- where: {- id: payoutToUse.id,- },- data: {- amount: {- increment: totalEarnings,- },- periodEnd,- },- });- }+ await qstash.batchJSON(jobs);- totalProcessed++;- }),- );- console.log(`Processed chunk ${i + 1} of ${chunks.length}`);- }-- const successRate =- (totalProcessed / partnerProgramCommissionsArray.length) * 100;- console.log(- `Processed ${totalProcessed}/${partnerProgramCommissionsArray.length} partners with due commissions for partner groups with holding period days: ${holdingPeriodDays} (${successRate.toFixed(1)}% success rate)`,- );- }-- if (holdingPeriodsWithMoreToProcess.length > 0) {console.log(- `Several holding periods still have more due commissions: ${holdingPeriodsWithMoreToProcess.join(", ")}`,- );-- const qstashResponse = await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`,- body: programId ? { programId } : {}, // pass programId if defined, else pass an empty object- });-- if (!qstashResponse.messageId) {- throw new Error(- `Error sending message to Qstash to schedule next batch of payouts: ${JSON.stringify(qstashResponse)}`,- );- }-- return logAndRespond(- "Finished aggregating due commissions into payouts for current batch. Scheduling next batch...",+ `Enqueued index ${index + 1}/${programIdChunks.length} jobs to be processed.`,);}@@ -278,5 +61,3 @@ export const GET = withCron(async ({ rawBody }) => {"Finished aggregating due commissions into payouts for all batches.",);});--export const POST = GET;diff --git a/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.ts b/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.tsindex eea3c1e018..62542a1ed7 100644--- a/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.ts+++ b/apps/web/lib/actions/partners/trigger-aggregate-due-commissions.ts@@ -3,9 +3,10 @@ import { APP_DOMAIN_WITH_NGROK, prettyPrint } from "@dub/utils";export async function triggerAggregateDueCommissionsCronJob(programId: string) {const qstashResponse = await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`,+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions/process`,+ label: "aggregate-due-commissions",flowControl: {- key: programId,+ key: `aggregate-due-commissions-${programId}`,parallelism: 1,},body: {diff --git a/apps/web/scripts/programs/backfill-reuse-commission.ts b/apps/web/scripts/programs/backfill-reuse-commission.tsindex cc4091c768..4ce58e1bd2 100644--- a/apps/web/scripts/programs/backfill-reuse-commission.ts+++ b/apps/web/scripts/programs/backfill-reuse-commission.ts@@ -1,9 +1,9 @@+import { triggerAggregateDueCommissionsCronJob } from "@/lib/actions/partners/trigger-aggregate-due-commissions";import { isFirstConversion } from "@/lib/analytics/is-first-conversion";import { createId } from "@/lib/api/create-id";import { updateLinkStatsForImporter } from "@/lib/api/links/update-link-stats-for-importer";import { syncPartnerLinksStats } from "@/lib/api/partners/sync-partner-links-stats";import { executeWorkflows } from "@/lib/api/workflows/execute-workflows";-import { qstash } from "@/lib/cron";import { queuePartnerCommissionCreation } from "@/lib/partners/queue-partner-commission-creation";import { prisma } from "@/lib/prisma";import { getCustomerEventsTB } from "@/lib/tinybird/get-customer-events-tb";@@ -16,7 +16,7 @@ import { recordSaleWithTimestamp } from "@/lib/tinybird/record-sale";import { CreatePartnerCommissionProps } from "@/lib/types";import { leadEventSchemaTB } from "@/lib/zod/schemas/leads";import { saleEventSchemaTB } from "@/lib/zod/schemas/sales";-import { APP_DOMAIN_WITH_NGROK, nanoid, prettyPrint } from "@dub/utils";+import { nanoid } from "@dub/utils";import "dotenv-flow/config";import * as z from "zod/v4";@@ -343,13 +343,7 @@ async function main() {]);}- const qstashResponse = await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/payouts/aggregate-due-commissions`,- body: { programId },- });- console.log(- `Triggered aggregate due commissions cron job for program ${programId}: ${prettyPrint(qstashResponse)}`,- );+ await triggerAggregateDueCommissionsCronJob(programId);}main();