The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Refactor partner program summary cron job handling
Kiran K6bd35e8f20b0
diff --git a/apps/web/app/(ee)/api/cron/partner-program-summary/process/route.ts b/apps/web/app/(ee)/api/cron/partner-program-summary/process/route.tsnew file mode 100644index 0000000000..09616db3af--- /dev/null+++ b/apps/web/app/(ee)/api/cron/partner-program-summary/process/route.ts@@ -0,0 +1,330 @@+import { getAnalytics } from "@/lib/analytics/get-analytics";+import { qstash } from "@/lib/cron";+import { withCron } from "@/lib/cron/with-cron";+import { sendBatchEmail } from "@dub/email";+import PartnerProgramSummary from "@dub/email/templates/partner-program-summary";+import { prisma } from "@dub/prisma";+import { Prisma } from "@dub/prisma/client";+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";+import { endOfMonth, format, startOfMonth, subMonths } from "date-fns";+import { z } from "zod";+import { logAndRespond } from "../../utils";++export const dynamic = "force-dynamic";++const PARTNER_BATCH_SIZE = 100;++const queue = qstash.queue({+ queueName: "partner-program-summary",+});++const schema = z.object({+ programId: z.string(),+ startingAfter: z.string().nullish(),+});++interface AnalyticsResponse {+ partnerId: string;+ clicks: number;+ leads: number;+ sales: number;+ saleAmount: number;+}++// POST /api/cron/partner-program-summary/process+export const POST = withCron(async ({ rawBody }) => {+ const result = schema.parse(JSON.parse(rawBody));++ let { programId, startingAfter } = result;++ const previousMonth = startOfMonth(subMonths(new Date(), 2));+ const currentMonth = startOfMonth(subMonths(new Date(), 1));++ const program = await prisma.program.findUnique({+ where: {+ id: programId,+ },+ select: {+ id: true,+ name: true,+ logo: true,+ slug: true,+ supportEmail: true,+ workspaceId: true,+ },+ });++ if (!program) {+ return logAndRespond(`Program ${programId} not found.`);+ }++ console.info(`Sending program summary for ${program.slug}`, {+ previousMonth,+ currentMonth,+ });++ // Find the clicks, leads, sales analytics+ const [previousMonthAnalytics, currentMonthAnalytics] = await Promise.all([+ // 2 months ago+ getAnalytics({+ event: "composite",+ groupBy: "top_partners",+ workspaceId: program.workspaceId,+ programId: program.id,+ start: previousMonth,+ end: endOfMonth(previousMonth),+ }),++ // 1 month ago+ getAnalytics({+ event: "composite",+ groupBy: "top_partners",+ workspaceId: program.workspaceId,+ programId: program.id,+ start: currentMonth,+ end: endOfMonth(currentMonth),+ }),+ ]);++ const programEnrollments = await prisma.programEnrollment.findMany({+ where: {+ programId: program.id,+ status: "approved",+ partner: {+ users: {+ some: {},+ },+ },+ links: {+ some: {+ leads: {+ gt: 0,+ },+ },+ },+ },+ select: {+ id: true,+ partner: {+ select: {+ id: true,+ email: true,+ createdAt: true,+ },+ },+ links: {+ select: {+ clicks: true,+ leads: true,+ sales: true,+ },+ },+ },+ ...(startingAfter && {+ skip: 1,+ cursor: {+ id: startingAfter,+ },+ }),+ orderBy: {+ id: "desc",+ },+ take: PARTNER_BATCH_SIZE,+ });++ console.info(+ `Found ${programEnrollments.length} active partners that have signed up for partners.dub.co and have links with at least 1 total lead.`,+ );++ if (programEnrollments.length === 0) {+ return logAndRespond(+ `No more active partners found for program ${program.id}.`,+ );+ }++ // Find the earnings+ const partners = programEnrollments.map(({ partner }) => partner);++ const commissionWhere: Prisma.CommissionWhereInput = {+ earnings: {+ gt: 0,+ },+ programId: program.id,+ partnerId: {+ in: partners.map((partner) => partner.id),+ },+ status: {+ in: ["pending", "processed", "paid"],+ },+ };++ const [previousMonthEarnings, currentMonthEarnings, lifetimeEarnings] =+ await Promise.all([+ // Earnings 2 months ago (to compare with previous month)+ prisma.commission.groupBy({+ by: ["partnerId"],+ where: {+ ...commissionWhere,+ createdAt: {+ gte: previousMonth,+ lte: endOfMonth(previousMonth),+ },+ },+ _sum: {+ earnings: true,+ },+ }),++ // Earnings 1 month ago,+ prisma.commission.groupBy({+ by: ["partnerId"],+ where: {+ ...commissionWhere,+ createdAt: {+ gte: currentMonth,+ lte: endOfMonth(currentMonth),+ },+ },+ _sum: {+ earnings: true,+ },+ }),++ // All-time earnings+ prisma.commission.groupBy({+ by: ["partnerId"],+ where: {+ ...commissionWhere,+ },+ _sum: {+ earnings: true,+ },+ }),+ ]);++ const previousEarningsMap = new Map(+ previousMonthEarnings.map((e) => [e.partnerId, e]),+ );++ const currentEarningsMap = new Map(+ currentMonthEarnings.map((e) => [e.partnerId, e]),+ );++ const lifetimeEarningsMap = new Map(+ lifetimeEarnings.map((e) => [e.partnerId, e]),+ );++ const previousAnalyticsMap: Map<string, AnalyticsResponse> = new Map(+ previousMonthAnalytics.map((a: AnalyticsResponse) => [a.partnerId, a]),+ );++ const currentAnalyticsMap: Map<string, AnalyticsResponse> = new Map(+ currentMonthAnalytics.map((a: AnalyticsResponse) => [a.partnerId, a]),+ );++ const summary = partners.map((partner) => {+ // Get previous and current month analytics from Tinybird+ const _previousMonthAnalytics = previousAnalyticsMap.get(partner.id);+ const _currentMonthAnalytics = currentAnalyticsMap.get(partner.id);++ // Get lifetime analytics from MySQL+ const _lifetimeAnalytics = programEnrollments+ .find((enrollment) => enrollment.partner.id === partner.id)+ ?.links.reduce(+ (acc, link) => ({+ clicks: acc.clicks + link.clicks,+ leads: acc.leads + link.leads,+ sales: acc.sales + link.sales,+ }),+ { clicks: 0, leads: 0, sales: 0 },+ );++ // Get earnings data from MySQL+ const _previousMonthEarnings = previousEarningsMap.get(partner.id);+ const _currentMonthEarnings = currentEarningsMap.get(partner.id);+ const _lifetimeEarnings = lifetimeEarningsMap.get(partner.id);++ return {+ partner,+ previousMonth: {+ clicks: _previousMonthAnalytics?.clicks ?? 0,+ leads: _previousMonthAnalytics?.leads ?? 0,+ sales: _previousMonthAnalytics?.sales ?? 0,+ earnings: _previousMonthEarnings?._sum.earnings ?? 0,+ },+ currentMonth: {+ clicks: _currentMonthAnalytics?.clicks ?? 0,+ leads: _currentMonthAnalytics?.leads ?? 0,+ sales: _currentMonthAnalytics?.sales ?? 0,+ earnings: _currentMonthEarnings?._sum.earnings ?? 0,+ },+ lifetime: {+ clicks: _lifetimeAnalytics?.clicks ?? 0,+ leads: _lifetimeAnalytics?.leads ?? 0,+ sales: _lifetimeAnalytics?.sales ?? 0,+ earnings: _lifetimeEarnings?._sum.earnings ?? 0,+ },+ };+ });++ console.table(+ summary.map((s) => ({+ partner: s.partner.email,+ program: program.name,+ currentClicks: s.currentMonth.clicks,+ currentLeads: s.currentMonth.leads,+ currentSales: s.currentMonth.sales,+ currentEarnings: s.currentMonth.earnings,+ lifetimeClicks: s.lifetime.clicks,+ lifetimeLeads: s.lifetime.leads,+ lifetimeSales: s.lifetime.sales,+ lifetimeEarnings: s.lifetime.earnings,+ })),+ );++ const reportingMonth = format(currentMonth, "MMM yyyy");++ await sendBatchEmail(+ summary.map(({ partner, ...rest }) => ({+ variant: "notifications",+ subject: `Your ${reportingMonth} performance report for ${program.name} program`,+ to: partner.email!,+ replyTo: program.supportEmail || "noreply",+ react: PartnerProgramSummary({+ program,+ partner,+ ...rest,+ reportingPeriod: {+ month: reportingMonth,+ start: currentMonth.toISOString(),+ end: endOfMonth(currentMonth).toISOString(),+ },+ }),+ })),+ {+ idempotencyKey: `partner-program-summary-${reportingMonth}-${program.id}`,+ },+ );++ // Schedule the next batch if there are more partners to process+ if (programEnrollments.length === PARTNER_BATCH_SIZE) {+ startingAfter = programEnrollments[programEnrollments.length - 1].id;++ const response = await queue.enqueueJSON({+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-program-summary/process`,+ method: "POST",+ body: {+ ...result,+ startingAfter,+ },+ });++ return logAndRespond(+ `Enqueued partner program summary jobs for the next batch ${response.messageId}`,+ );+ }++ return logAndRespond(+ `Finished processing all partners for program ${program.id}.`,+ );+});diff --git a/apps/web/app/(ee)/api/cron/partner-program-summary/route.ts b/apps/web/app/(ee)/api/cron/partner-program-summary/route.tsindex c93d54734f..bba7a1cd5d 100644--- a/apps/web/app/(ee)/api/cron/partner-program-summary/route.ts+++ b/apps/web/app/(ee)/api/cron/partner-program-summary/route.ts@@ -1,365 +1,54 @@-import { getAnalytics } from "@/lib/analytics/get-analytics";-import { handleAndReturnErrorResponse } from "@/lib/api/errors";-import { qstash } from "@/lib/cron";-import { verifyQstashSignature } from "@/lib/cron/verify-qstash";-import { verifyVercelSignature } from "@/lib/cron/verify-vercel";-import { sendBatchEmail } from "@dub/email";-import PartnerProgramSummary from "@dub/email/templates/partner-program-summary";+import { enqueueBatchJobs } from "@/lib/cron/enqueue-batch-jobs";+import { withCron } from "@/lib/cron/with-cron";import { prisma } from "@dub/prisma";-import { APP_DOMAIN_WITH_NGROK, log } from "@dub/utils";-import { Prisma } from "@prisma/client";-import { endOfMonth, format, startOfMonth, subMonths } from "date-fns";-import { NextResponse } from "next/server";-import { z } from "zod";--/**- * This route handles the monthly partner program summary emails for partners.- * Scheduled to run at 1 PM UTC on the 1st day of every month to send the previous month's summary.- *- * 1. Processing Flow:- * - Processes one program at a time (paginated)- * - For each program, processes partners in batches of 1- * - Only processes approved partners with active leads- *- * 2. Data Collection:- * - Gathers analytics for three time periods:- * * Current month (1 month ago) - current reporting period (from Tinybird)- * * Previous month (2 months ago) - previous reporting period for comparison (from Tinybird)- * * Lifetime - all-time stats (from MySQL)- * - Collects metrics: clicks, leads, sales, and earnings- */+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";+import { format, startOfMonth, subMonths } from "date-fns";+import { logAndRespond } from "../utils";export const dynamic = "force-dynamic";-type AnalyticsResponse = {- partnerId: string;- clicks: number;- leads: number;- sales: number;- saleAmount: number;-};--const schema = z.object({- skip: z.number().optional().default(0),-});+const PROGRAM_BATCH_SIZE = 50;-// GET/POST /api/cron/partner-program-summary-async function handler(req: Request) {- try {- let programSkip = 0;- const programsTake = 1;+// GET /api/cron/partner-program-summary+export const GET = withCron(async () => {+ const currentMonth = startOfMonth(subMonths(new Date(), 1));+ const yearMonth = format(currentMonth, "yyyy-MM");- if (req.method === "GET") {- await verifyVercelSignature(req);- } else if (req.method === "POST") {- const rawBody = await req.text();- const result = schema.parse(JSON.parse(rawBody));- programSkip = result.skip;+ console.log(yearMonth, currentMonth);- await verifyQstashSignature({- req,- rawBody,- });- }+ let page = 0;+ while (true) {const programs = await prisma.program.findMany({select: {id: true,- name: true,- logo: true,- slug: true,- supportEmail: true,- workspaceId: true,},- skip: programSkip,- take: programsTake,+ take: PROGRAM_BATCH_SIZE,+ skip: page * PROGRAM_BATCH_SIZE,orderBy: {- createdAt: "asc",+ id: "asc",},});if (programs.length === 0) {- console.log("No more programs found.");- return NextResponse.json("No more programs found.");+ break;}- // Since a program can have many partners, we process only one program at a time- const program = programs[0];- programSkip += programsTake;-- const previousMonth = startOfMonth(subMonths(new Date(), 2));- const currentMonth = startOfMonth(subMonths(new Date(), 1));-- console.log(`Sending program summary for ${program.slug}`, {- previousMonth,- currentMonth,- });-- // Find the clicks, leads, sales analytics- const [previousMonthAnalytics, currentMonthAnalytics] = await Promise.all([- // 2 months ago- getAnalytics({- event: "composite",- groupBy: "top_partners",- workspaceId: program.workspaceId,- programId: program.id,- start: previousMonth,- end: endOfMonth(previousMonth),- }),-- // 1 month ago- getAnalytics({- event: "composite",- groupBy: "top_partners",- workspaceId: program.workspaceId,- programId: program.id,- start: currentMonth,- end: endOfMonth(currentMonth),- }),- ]);-- let partnersSkip = 0;- const partnersTake = 100;-- /*- Process 100 partners at a time for a program that match the criteria:- - is currently approved- - has signed up for partners.dub.co- - has links that have at least 1 total lead- */- while (true) {- const programEnrollments = await prisma.programEnrollment.findMany({- where: {+ await enqueueBatchJobs(+ programs.map((program) => ({+ queueName: "partner-program-summary",+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-program-summary/process`,+ deduplicationId: `partner-program-summary-${yearMonth}-${program.id}`,+ body: {programId: program.id,- status: "approved",- partner: {- users: {- some: {},- },- },- links: {- some: {- leads: {- gt: 0,- },- },- },},- select: {- partner: {- select: {- id: true,- email: true,- createdAt: true,- },- },- links: {- select: {- clicks: true,- leads: true,- sales: true,- },- },- },- skip: partnersSkip,- take: partnersTake,- orderBy: {- createdAt: "asc",- },- });-- console.log(- `Found ${programEnrollments.length} active partners that have signed up for partners.dub.co and have links with at least 1 total lead.`,- );-- if (programEnrollments.length === 0) {- console.log(`No more active partners found for program ${program.id}.`);- break;- }-- partnersSkip += partnersTake;-- // Find the earnings- const partners = programEnrollments.map(- (enrollment) => enrollment.partner,- );-- const commissionWhere: Prisma.CommissionWhereInput = {- earnings: {- gt: 0,- },- programId: program.id,- partnerId: {- in: partners.map((partner) => partner.id),- },- status: {- in: ["pending", "processed", "paid"],- },- };-- const [previousMonthEarnings, currentMonthEarnings, lifetimeEarnings] =- await Promise.all([- // Earnings 2 months ago (to compare with previous month)- prisma.commission.groupBy({- by: ["partnerId"],- where: {- ...commissionWhere,- createdAt: {- gte: previousMonth,- lte: endOfMonth(previousMonth),- },- },- _sum: {- earnings: true,- },- }),+ })),+ );- // Earnings 1 month ago,- prisma.commission.groupBy({- by: ["partnerId"],- where: {- ...commissionWhere,- createdAt: {- gte: currentMonth,- lte: endOfMonth(currentMonth),- },- },- _sum: {- earnings: true,- },- }),-- // All-time earnings- prisma.commission.groupBy({- by: ["partnerId"],- where: {- ...commissionWhere,- },- _sum: {- earnings: true,- },- }),- ]);-- const previousEarningsMap = new Map(- previousMonthEarnings.map((e) => [e.partnerId, e]),- );-- const currentEarningsMap = new Map(- currentMonthEarnings.map((e) => [e.partnerId, e]),- );-- const lifetimeEarningsMap = new Map(- lifetimeEarnings.map((e) => [e.partnerId, e]),- );-- const previousAnalyticsMap: Map<string, AnalyticsResponse> = new Map(- previousMonthAnalytics.map((a) => [a.partnerId, a]),- );-- const currentAnalyticsMap: Map<string, AnalyticsResponse> = new Map(- currentMonthAnalytics.map((a) => [a.partnerId, a]),- );-- let summary = partners.map((partner) => {- // get previous and current month analytics from Tinybird- const _previousMonthAnalytics = previousAnalyticsMap.get(partner.id);- const _currentMonthAnalytics = currentAnalyticsMap.get(partner.id);- // get lifetime analytics from MySQL- const _lifetimeAnalytics = programEnrollments- .find((enrollment) => enrollment.partner.id === partner.id)- ?.links.reduce((acc, link) => ({- clicks: acc.clicks + link.clicks,- leads: acc.leads + link.leads,- sales: acc.sales + link.sales,- }));-- // get earnings data from MySQL- const _previousMonthEarnings = previousEarningsMap.get(partner.id);- const _currentMonthEarnings = currentEarningsMap.get(partner.id);- const _lifetimeEarnings = lifetimeEarningsMap.get(partner.id);-- return {- partner,- previousMonth: {- clicks: _previousMonthAnalytics?.clicks ?? 0,- leads: _previousMonthAnalytics?.leads ?? 0,- sales: _previousMonthAnalytics?.sales ?? 0,- earnings: _previousMonthEarnings?._sum.earnings ?? 0,- },- currentMonth: {- clicks: _currentMonthAnalytics?.clicks ?? 0,- leads: _currentMonthAnalytics?.leads ?? 0,- sales: _currentMonthAnalytics?.sales ?? 0,- earnings: _currentMonthEarnings?._sum.earnings ?? 0,- },- lifetime: {- clicks: _lifetimeAnalytics?.clicks ?? 0,- leads: _lifetimeAnalytics?.leads ?? 0,- sales: _lifetimeAnalytics?.sales ?? 0,- earnings: _lifetimeEarnings?._sum.earnings ?? 0,- },- };- });-- console.log(`Found ${summary.length} partners to send summary to.`);- console.table(- summary.map((s) => ({- partner: s.partner.email,- program: program.name,- currentClicks: s.currentMonth.clicks,- currentLeads: s.currentMonth.leads,- currentSales: s.currentMonth.sales,- currentEarnings: s.currentMonth.earnings,- lifetimeClicks: s.lifetime.clicks,- lifetimeLeads: s.lifetime.leads,- lifetimeSales: s.lifetime.sales,- lifetimeEarnings: s.lifetime.earnings,- })),- );-- const reportingMonth = format(currentMonth, "MMM yyyy");-- await sendBatchEmail(- summary.map(({ partner, ...rest }) => ({- subject: `Your ${reportingMonth} performance report for ${program.name} program`,- to: partner.email!,- replyTo: program.supportEmail || "noreply",- react: PartnerProgramSummary({- program,- partner,- ...rest,- reportingPeriod: {- month: reportingMonth,- start: currentMonth.toISOString(),- end: endOfMonth(currentMonth).toISOString(),- },- }),- variant: "notifications",- })),- {- idempotencyKey: `partner-summary/${program.id}/${format(currentMonth, "yyyy-MM")}`,- },- );- }-- await qstash.publishJSON({- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-program-summary`,- method: "POST",- body: {- skip: programSkip,- },- });-- return NextResponse.json("Ok");- } catch (error) {- await log({- message: `Error sending partner program summary: ${error.message}`,- type: "cron",- });-- return handleAndReturnErrorResponse(error);+ page++;}-}-export { handler as GET, handler as POST };+ return logAndRespond(+ `Enqueued partner program summary jobs for ${yearMonth}.`,+ );+});diff --git a/apps/web/lib/cron/enqueue-batch-jobs.ts b/apps/web/lib/cron/enqueue-batch-jobs.tsindex 7358edbdd6..e6de94d99e 100644--- a/apps/web/lib/cron/enqueue-batch-jobs.ts+++ b/apps/web/lib/cron/enqueue-batch-jobs.ts@@ -1,20 +1,24 @@import { log } from "@dub/utils";+import type { PublishBatchRequest } from "@upstash/qstash";import { qstash } from ".";-interface BatchJob<T> {- queueName: string;- url: string;- body: T;-}+type EnqueueBatchJobsProps = PublishBatchRequest<unknown> & {+ queueName: "ban-partner" | "partner-program-summary";+};// Generic helper to enqueue a batch of QStash jobs.-export async function enqueueBatchJobs<T>(jobs: BatchJob<T>[]) {+export async function enqueueBatchJobs(jobs: EnqueueBatchJobsProps[]) {try {const result = await qstash.batchJSON(jobs);- console.info(- `[enqueueBatchJobs] ${result.length} batch jobs enqueued successfully`,- );+ if (process.env.NODE_ENV === "development") {+ console.info(+ `[enqueueBatchJobs] ${result.length} batch jobs enqueued successfully.`,+ {+ jobs,+ },+ );+ }return result;} catch (error) {@@ -29,6 +33,8 @@ export async function enqueueBatchJobs<T>(jobs: BatchJob<T>[]) {mention: true,});- return [];+ throw new Error(+ `Failed to enqueue batch jobs: ${JSON.stringify(error, null, 2)}`,+ );}}diff --git a/apps/web/lib/cron/with-cron.ts b/apps/web/lib/cron/with-cron.tsnew file mode 100644index 0000000000..e5be2841e3--- /dev/null+++ b/apps/web/lib/cron/with-cron.ts@@ -0,0 +1,67 @@+import { APP_DOMAIN_WITH_NGROK, getSearchParams, log } from "@dub/utils";+import { logAndRespond } from "app/(ee)/api/cron/utils";+import { withAxiomBodyLog } from "../axiom/server";+import { verifyQstashSignature } from "./verify-qstash";+import { verifyVercelSignature } from "./verify-vercel";++interface WithCronHandler {+ ({+ req,+ params,+ searchParams,+ rawBody,+ }: {+ req: Request;+ params: Record<string, string>;+ searchParams: Record<string, string>;+ rawBody: string;+ }): Promise<Response>;+}++export const withCron = (handler: WithCronHandler) => {+ return withAxiomBodyLog(+ async (+ req,+ { params: initialParams }: { params: Promise<Record<string, string>> },+ ) => {+ // Clone the request early so handlers can read the body without cloning+ // Keep the original for withAxiomBodyLog to read in onSuccess+ const clonedReq = req.clone();++ const params = (await initialParams) || {};+ const searchParams = getSearchParams(req.url);+ const url = new URL(req.url || "", APP_DOMAIN_WITH_NGROK);++ try {+ let rawBody: string | undefined;++ // Verify signature based on HTTP method+ // GET requests are typically from Vercel Cron+ if (req.method === "GET") {+ await verifyVercelSignature(req);+ }+ // POST requests are typically from QStash+ else if (req.method === "POST") {+ rawBody = await req.text();+ await verifyQstashSignature({ req, rawBody });+ }++ return await handler({+ req: clonedReq,+ searchParams,+ params,+ rawBody: rawBody ?? "",+ });+ } catch (error) {+ console.error(error);++ await log({+ message: `Cron job ${url.pathname} failed ${error.message}`,+ type: "cron",+ });++ return logAndRespond(error.message);+ }+ },+ );+};