The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Separate `importScheduledCommisisons` (#3688)
Steven Tey248dc3a8c80c
diff --git a/apps/web/app/(ee)/api/cron/import/partnerstack/route.ts b/apps/web/app/(ee)/api/cron/import/partnerstack/route.tsindex 3e1c1f1048..765a154b91 100644--- a/apps/web/app/(ee)/api/cron/import/partnerstack/route.ts+++ b/apps/web/app/(ee)/api/cron/import/partnerstack/route.ts@@ -5,6 +5,7 @@ import { importCustomers } from "@/lib/partnerstack/import-customers";import { importGroups } from "@/lib/partnerstack/import-groups";import { importLinks } from "@/lib/partnerstack/import-links";import { importPartners } from "@/lib/partnerstack/import-partners";+import { importScheduledCommissions } from "@/lib/partnerstack/import-scheduled-commissions";import { partnerStackImportPayloadSchema } from "@/lib/partnerstack/schemas";import { updateStripeCustomers } from "@/lib/partnerstack/update-stripe-customers";import { NextResponse } from "next/server";@@ -38,6 +39,9 @@ export async function POST(req: Request) {case "import-commissions":await importCommissions(payload);break;+ case "import-scheduled-commissions":+ await importScheduledCommissions(payload);+ break;case "update-stripe-customers":await updateStripeCustomers(payload);break;diff --git a/apps/web/lib/firstpromoter/api.ts b/apps/web/lib/firstpromoter/api.tsindex 3ab91b81a7..c7925331e7 100644--- a/apps/web/lib/firstpromoter/api.ts+++ b/apps/web/lib/firstpromoter/api.ts@@ -1,5 +1,5 @@import * as z from "zod/v4";-import { PAGE_LIMIT } from "./importer";+import { FIRSTPROMOTER_PAGE_LIMIT } from "./importer";import {firstPromoterCampaignSchema,firstPromoterCommissionSchema,@@ -46,7 +46,7 @@ export class FirstPromoterApi {async listCampaigns({ page }: { page?: number }) {const searchParams = new URLSearchParams({- per_page: PAGE_LIMIT.toString(),+ per_page: FIRSTPROMOTER_PAGE_LIMIT.toString(),...(page ? { page: page.toString() } : {}),});@@ -61,7 +61,7 @@ export class FirstPromoterApi {const searchParams = new URLSearchParams();searchParams.set("filters[state]", "accepted");searchParams.set("filters[referrals_count][from]", "1");- searchParams.set("per_page", PAGE_LIMIT.toString());+ searchParams.set("per_page", FIRSTPROMOTER_PAGE_LIMIT.toString());if (page) searchParams.set("page", page.toString());const response = await this.fetch(`/promoters?${searchParams.toString()}`);@@ -77,7 +77,7 @@ export class FirstPromoterApi {async listCustomers({ page }: { page?: number }) {const searchParams = new URLSearchParams({- per_page: PAGE_LIMIT.toString(),+ per_page: FIRSTPROMOTER_PAGE_LIMIT.toString(),...(page ? { page: page.toString() } : {}),});@@ -95,7 +95,7 @@ export class FirstPromoterApi {async listCommissions({ page }: { page?: number }) {const searchParams = new URLSearchParams({- per_page: PAGE_LIMIT.toString(),+ per_page: FIRSTPROMOTER_PAGE_LIMIT.toString(),...(page ? { page: page.toString() } : {}),});diff --git a/apps/web/lib/firstpromoter/importer.ts b/apps/web/lib/firstpromoter/importer.tsindex 11a6fb6b49..aa722fa63a 100644--- a/apps/web/lib/firstpromoter/importer.ts+++ b/apps/web/lib/firstpromoter/importer.ts@@ -3,7 +3,7 @@ import { redis } from "@/lib/upstash";import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";import { FirstPromoterCredentials, FirstPromoterImportPayload } from "./types";-export const PAGE_LIMIT = 100;+export const FIRSTPROMOTER_PAGE_LIMIT = 100;export const MAX_BATCHES = 10;export const CACHE_EXPIRY = 60 * 60 * 24;export const CACHE_KEY_PREFIX = "firstpromoter:import";diff --git a/apps/web/lib/partnerstack/api.ts b/apps/web/lib/partnerstack/api.tsindex ddadb7a9bf..aaac4bc1cd 100644--- a/apps/web/lib/partnerstack/api.ts+++ b/apps/web/lib/partnerstack/api.ts@@ -14,7 +14,7 @@ import {PartnerStackPartner,} from "./types";-const PAGE_LIMIT = 100;+export const PARTNERSTACK_PAGE_LIMIT = 100;export class PartnerStackApi {private readonly baseUrl = "https://api.partnerstack.com/api/v2";@@ -77,7 +77,7 @@ export class PartnerStackApi {async listPartners({ startingAfter }: { startingAfter?: string }) {const searchParams = new URLSearchParams();searchParams.append("approved_status", "approved");- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", PARTNERSTACK_PAGE_LIMIT.toString());if (startingAfter) {searchParams.append("starting_after", startingAfter);@@ -104,7 +104,7 @@ export class PartnerStackApi {async listCustomers({ startingAfter }: { startingAfter?: string }) {const searchParams = new URLSearchParams();- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", PARTNERSTACK_PAGE_LIMIT.toString());if (startingAfter) {searchParams.append("starting_after", startingAfter);@@ -127,7 +127,7 @@ export class PartnerStackApi {status?: PartnerStackCommission["reward_status"];}) {const searchParams = new URLSearchParams();- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", PARTNERSTACK_PAGE_LIMIT.toString());if (startingAfter) {searchParams.append("starting_after", startingAfter);diff --git a/apps/web/lib/partnerstack/import-commissions.ts b/apps/web/lib/partnerstack/import-commissions.tsindex 1fa2c6368a..dcb877db6b 100644--- a/apps/web/lib/partnerstack/import-commissions.ts+++ b/apps/web/lib/partnerstack/import-commissions.ts@@ -103,7 +103,7 @@ export async function importCommissions(payload: PartnerStackImportPayload) {await Promise.allSettled(commissions.map((commission) =>- createCommission({+ createCommissionFromPS({program,commission,fxRates,@@ -114,86 +114,20 @@ export async function importCommissions(payload: PartnerStackImportPayload) {),);- await new Promise((resolve) => setTimeout(resolve, 1000));-currentStartingAfter = commissions[commissions.length - 1].key;processedBatches++;}- // Import scheduled commissions- if (!hasMore) {- const commissions = await partnerStackApi.listCommissions({- status: "scheduled",- });-- if (commissions.length > 0) {- const customersData = await prisma.customer.findMany({- where: {- projectId: program.workspaceId,- OR: [- {- email: {- in: commissions- .map((commission) => commission.customer?.email)- .filter(- (email): email is string =>- email !== null && email !== undefined,- ),- },- },- {- externalId: {- in: commissions- .map((commission) => commission.customer?.external_key)- .filter(- (externalKey): externalKey is string =>- externalKey !== null && externalKey !== undefined,- ),- },- },- ],- },- include: {- link: true,- },- orderBy: {- createdAt: "asc",- },- });-- const customerLeadEvents = await getLeadEvents({- customerIds: customersData.map((customer) => customer.id),- }).then((res) => res.data);-- await Promise.allSettled(- commissions.map((commission) =>- createCommission({- program,- commission,- fxRates,- importId,- customersData,- customerLeadEvents,- }),- ),- );-- await new Promise((resolve) => setTimeout(resolve, 1000));- }- }-- if (!hasMore) {- await partnerStackImporter.deleteCredentials(program.workspaceId);- }+ await new Promise((resolve) => setTimeout(resolve, 1000));await partnerStackImporter.queue({...payload,startingAfter: hasMore ? currentStartingAfter : undefined,- action: hasMore ? "import-commissions" : "update-stripe-customers",+ action: hasMore ? "import-commissions" : "import-scheduled-commissions",});}-async function createCommission({+export async function createCommissionFromPS({program,commission,fxRates,diff --git a/apps/web/lib/partnerstack/import-scheduled-commissions.ts b/apps/web/lib/partnerstack/import-scheduled-commissions.tsnew file mode 100644index 0000000000..bbe72144da--- /dev/null+++ b/apps/web/lib/partnerstack/import-scheduled-commissions.ts@@ -0,0 +1,117 @@+import { prisma } from "@dub/prisma";+import { getLeadEvents } from "../tinybird/get-lead-events";+import { redis } from "../upstash";+import { PartnerStackApi } from "./api";+import { createCommissionFromPS } from "./import-commissions";+import { MAX_BATCHES, partnerStackImporter } from "./importer";+import { PartnerStackImportPayload } from "./types";++export async function importScheduledCommissions(+ payload: PartnerStackImportPayload,+) {+ const { importId, programId, startingAfter } = payload;++ const program = await prisma.program.findUniqueOrThrow({+ where: {+ id: programId,+ },+ });++ const { publicKey, secretKey } = await partnerStackImporter.getCredentials(+ program.workspaceId,+ );++ const partnerStackApi = new PartnerStackApi({+ publicKey,+ secretKey,+ });++ const fxRates = await redis.hgetall<Record<string, string>>("fxRates:usd");++ let hasMore = true;+ let processedBatches = 0;+ let currentStartingAfter = startingAfter;++ while (hasMore && processedBatches < MAX_BATCHES) {+ const scheduledCommissions = await partnerStackApi.listCommissions({+ status: "scheduled",+ startingAfter: currentStartingAfter,+ });++ if (scheduledCommissions.length === 0) {+ hasMore = false;+ break;+ }++ const customersData = await prisma.customer.findMany({+ where: {+ projectId: program.workspaceId,+ OR: [+ {+ email: {+ in: scheduledCommissions+ .map((commission) => commission.customer?.email)+ .filter(+ (email): email is string =>+ email !== null && email !== undefined,+ ),+ },+ },+ {+ externalId: {+ in: scheduledCommissions+ .map((commission) => commission.customer?.external_key)+ .filter(+ (externalKey): externalKey is string =>+ externalKey !== null && externalKey !== undefined,+ ),+ },+ },+ ],+ },+ include: {+ link: true,+ },+ orderBy: {+ createdAt: "asc",+ },+ });++ const customerLeadEvents = await getLeadEvents({+ customerIds: customersData.map((customer) => customer.id),+ }).then((res) => res.data);++ await Promise.allSettled(+ scheduledCommissions.map((commission) =>+ createCommissionFromPS({+ program,+ commission,+ fxRates,+ importId,+ customersData,+ customerLeadEvents,+ }),+ ),+ );++ currentStartingAfter =+ scheduledCommissions[scheduledCommissions.length - 1].key;+ processedBatches++;+ }++ // If there are more scheduled commissions to import, sleep for 1 second and continue the loop+ if (hasMore) {+ await new Promise((resolve) => setTimeout(resolve, 1000));+ } else {+ // else, delete the credentials and exit the loop+ await partnerStackImporter.deleteCredentials(program.workspaceId);+ }++ await partnerStackImporter.queue({+ ...payload,+ startingAfter: hasMore ? currentStartingAfter : undefined,+ action: hasMore+ ? "import-scheduled-commissions"+ : "update-stripe-customers",+ });+}diff --git a/apps/web/lib/partnerstack/schemas.ts b/apps/web/lib/partnerstack/schemas.tsindex dd1a253c95..13872553cf 100644--- a/apps/web/lib/partnerstack/schemas.ts+++ b/apps/web/lib/partnerstack/schemas.ts@@ -6,6 +6,7 @@ export const partnerStackImportSteps = z.enum(["import-links","import-customers","import-commissions",+ "import-scheduled-commissions","update-stripe-customers",]);diff --git a/apps/web/lib/rewardful/api.ts b/apps/web/lib/rewardful/api.tsindex 6335e85800..e919b3cd36 100644--- a/apps/web/lib/rewardful/api.ts+++ b/apps/web/lib/rewardful/api.ts@@ -7,7 +7,7 @@ import {RewardfulReferral,} from "./types";-const PAGE_LIMIT = 100;+const REWARDFUL_PAGE_LIMIT = 100;class RewardfulApiError extends DubApiError {constructor(message: string) {@@ -57,7 +57,7 @@ export class RewardfulApi {searchParams.append("expand[]", "campaign");searchParams.append("expand[]", "links");searchParams.append("page", page.toString());- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());const { data } = await this.fetch<{ data: RewardfulAffiliate[] }>(`${this.baseUrl}/affiliates?${searchParams.toString()}`,@@ -72,7 +72,7 @@ export class RewardfulApi {searchParams.append("conversion_state[]", "lead");searchParams.append("conversion_state[]", "conversion");searchParams.append("page", page.toString());- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());const { data } = await this.fetch<{ data: RewardfulReferral[] }>(`${this.baseUrl}/referrals?${searchParams.toString()}`,@@ -86,7 +86,7 @@ export class RewardfulApi {searchParams.append("expand[]", "sale");searchParams.append("expand[]", "campaign");searchParams.append("page", page.toString());- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());const { data } = await this.fetch<{ data: RewardfulCommission[] }>(`${this.baseUrl}/commissions?${searchParams.toString()}`,@@ -98,7 +98,7 @@ export class RewardfulApi {async listAffiliateCoupons({ page = 1 }: { page?: number }) {const searchParams = new URLSearchParams();searchParams.append("page", page.toString());- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());const { data } = await this.fetch<{ data: RewardfulCoupon[] }>(`${this.baseUrl}/affiliate_coupons?${searchParams.toString()}`,diff --git a/apps/web/lib/tolt/api.ts b/apps/web/lib/tolt/api.tsindex 8f39a9ccbe..9748104d96 100644--- a/apps/web/lib/tolt/api.ts+++ b/apps/web/lib/tolt/api.ts@@ -13,7 +13,7 @@ import {ToltListResponse,} from "./types";-const PAGE_LIMIT = 100;+const TOLT_PAGE_LIMIT = 100;export class ToltApi {private readonly baseUrl = "https://api.tolt.com/v1";@@ -80,7 +80,7 @@ export class ToltApi {const searchParams = new URLSearchParams();searchParams.append("program_id", programId);searchParams.append("expand[]", "program");- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());if (startingAfter) {searchParams.append("starting_after", startingAfter);@@ -103,7 +103,7 @@ export class ToltApi {const searchParams = new URLSearchParams();searchParams.append("program_id", programId);searchParams.append("expand[]", "partner");- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());if (startingAfter) {searchParams.append("starting_after", startingAfter);@@ -126,7 +126,7 @@ export class ToltApi {const searchParams = new URLSearchParams();searchParams.append("program_id", programId);searchParams.append("expand[]", "partner");- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());if (startingAfter) {searchParams.append("starting_after", startingAfter);@@ -152,7 +152,7 @@ export class ToltApi {searchParams.append("expand[]", "partner");searchParams.append("expand[]", "customer");searchParams.append("expand[]", "transaction");- searchParams.append("limit", PAGE_LIMIT.toString());+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());if (startingAfter) {searchParams.append("starting_after", startingAfter);