Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Separate `importScheduledCommisisons` (#3688)

Steven Tey248dc3a8c80c
diff --git a/apps/web/app/(ee)/api/cron/import/partnerstack/route.ts b/apps/web/app/(ee)/api/cron/import/partnerstack/route.ts
index 3e1c1f1048..765a154b91 100644
--- a/apps/web/app/(ee)/api/cron/import/partnerstack/route.ts
+++ b/apps/web/app/(ee)/api/cron/import/partnerstack/route.ts
@@ -5,6 +5,7 @@ import { importCustomers } from "@/lib/partnerstack/import-customers";
import { importGroups } from "@/lib/partnerstack/import-groups";
import { importLinks } from "@/lib/partnerstack/import-links";
import { importPartners } from "@/lib/partnerstack/import-partners";
+import { importScheduledCommissions } from "@/lib/partnerstack/import-scheduled-commissions";
import { partnerStackImportPayloadSchema } from "@/lib/partnerstack/schemas";
import { updateStripeCustomers } from "@/lib/partnerstack/update-stripe-customers";
import { NextResponse } from "next/server";
@@ -38,6 +39,9 @@ export async function POST(req: Request) {
case "import-commissions":
await importCommissions(payload);
break;
+ case "import-scheduled-commissions":
+ await importScheduledCommissions(payload);
+ break;
case "update-stripe-customers":
await updateStripeCustomers(payload);
break;
diff --git a/apps/web/lib/firstpromoter/api.ts b/apps/web/lib/firstpromoter/api.ts
index 3ab91b81a7..c7925331e7 100644
--- a/apps/web/lib/firstpromoter/api.ts
+++ b/apps/web/lib/firstpromoter/api.ts
@@ -1,5 +1,5 @@
import * as z from "zod/v4";
-import { PAGE_LIMIT } from "./importer";
+import { FIRSTPROMOTER_PAGE_LIMIT } from "./importer";
import {
firstPromoterCampaignSchema,
firstPromoterCommissionSchema,
@@ -46,7 +46,7 @@ export class FirstPromoterApi {
async listCampaigns({ page }: { page?: number }) {
const searchParams = new URLSearchParams({
- per_page: PAGE_LIMIT.toString(),
+ per_page: FIRSTPROMOTER_PAGE_LIMIT.toString(),
...(page ? { page: page.toString() } : {}),
});
@@ -61,7 +61,7 @@ export class FirstPromoterApi {
const searchParams = new URLSearchParams();
searchParams.set("filters[state]", "accepted");
searchParams.set("filters[referrals_count][from]", "1");
- searchParams.set("per_page", PAGE_LIMIT.toString());
+ searchParams.set("per_page", FIRSTPROMOTER_PAGE_LIMIT.toString());
if (page) searchParams.set("page", page.toString());
const response = await this.fetch(`/promoters?${searchParams.toString()}`);
@@ -77,7 +77,7 @@ export class FirstPromoterApi {
async listCustomers({ page }: { page?: number }) {
const searchParams = new URLSearchParams({
- per_page: PAGE_LIMIT.toString(),
+ per_page: FIRSTPROMOTER_PAGE_LIMIT.toString(),
...(page ? { page: page.toString() } : {}),
});
@@ -95,7 +95,7 @@ export class FirstPromoterApi {
async listCommissions({ page }: { page?: number }) {
const searchParams = new URLSearchParams({
- per_page: PAGE_LIMIT.toString(),
+ per_page: FIRSTPROMOTER_PAGE_LIMIT.toString(),
...(page ? { page: page.toString() } : {}),
});
diff --git a/apps/web/lib/firstpromoter/importer.ts b/apps/web/lib/firstpromoter/importer.ts
index 11a6fb6b49..aa722fa63a 100644
--- a/apps/web/lib/firstpromoter/importer.ts
+++ b/apps/web/lib/firstpromoter/importer.ts
@@ -3,7 +3,7 @@ import { redis } from "@/lib/upstash";
import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
import { FirstPromoterCredentials, FirstPromoterImportPayload } from "./types";
-export const PAGE_LIMIT = 100;
+export const FIRSTPROMOTER_PAGE_LIMIT = 100;
export const MAX_BATCHES = 10;
export const CACHE_EXPIRY = 60 * 60 * 24;
export const CACHE_KEY_PREFIX = "firstpromoter:import";
diff --git a/apps/web/lib/partnerstack/api.ts b/apps/web/lib/partnerstack/api.ts
index ddadb7a9bf..aaac4bc1cd 100644
--- a/apps/web/lib/partnerstack/api.ts
+++ b/apps/web/lib/partnerstack/api.ts
@@ -14,7 +14,7 @@ import {
PartnerStackPartner,
} from "./types";
-const PAGE_LIMIT = 100;
+export const PARTNERSTACK_PAGE_LIMIT = 100;
export class PartnerStackApi {
private readonly baseUrl = "https://api.partnerstack.com/api/v2";
@@ -77,7 +77,7 @@ export class PartnerStackApi {
async listPartners({ startingAfter }: { startingAfter?: string }) {
const searchParams = new URLSearchParams();
searchParams.append("approved_status", "approved");
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", PARTNERSTACK_PAGE_LIMIT.toString());
if (startingAfter) {
searchParams.append("starting_after", startingAfter);
@@ -104,7 +104,7 @@ export class PartnerStackApi {
async listCustomers({ startingAfter }: { startingAfter?: string }) {
const searchParams = new URLSearchParams();
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", PARTNERSTACK_PAGE_LIMIT.toString());
if (startingAfter) {
searchParams.append("starting_after", startingAfter);
@@ -127,7 +127,7 @@ export class PartnerStackApi {
status?: PartnerStackCommission["reward_status"];
}) {
const searchParams = new URLSearchParams();
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", PARTNERSTACK_PAGE_LIMIT.toString());
if (startingAfter) {
searchParams.append("starting_after", startingAfter);
diff --git a/apps/web/lib/partnerstack/import-commissions.ts b/apps/web/lib/partnerstack/import-commissions.ts
index 1fa2c6368a..dcb877db6b 100644
--- a/apps/web/lib/partnerstack/import-commissions.ts
+++ b/apps/web/lib/partnerstack/import-commissions.ts
@@ -103,7 +103,7 @@ export async function importCommissions(payload: PartnerStackImportPayload) {
await Promise.allSettled(
commissions.map((commission) =>
- createCommission({
+ createCommissionFromPS({
program,
commission,
fxRates,
@@ -114,86 +114,20 @@ export async function importCommissions(payload: PartnerStackImportPayload) {
),
);
- await new Promise((resolve) => setTimeout(resolve, 1000));
-
currentStartingAfter = commissions[commissions.length - 1].key;
processedBatches++;
}
- // Import scheduled commissions
- if (!hasMore) {
- const commissions = await partnerStackApi.listCommissions({
- status: "scheduled",
- });
-
- if (commissions.length > 0) {
- const customersData = await prisma.customer.findMany({
- where: {
- projectId: program.workspaceId,
- OR: [
- {
- email: {
- in: commissions
- .map((commission) => commission.customer?.email)
- .filter(
- (email): email is string =>
- email !== null && email !== undefined,
- ),
- },
- },
- {
- externalId: {
- in: commissions
- .map((commission) => commission.customer?.external_key)
- .filter(
- (externalKey): externalKey is string =>
- externalKey !== null && externalKey !== undefined,
- ),
- },
- },
- ],
- },
- include: {
- link: true,
- },
- orderBy: {
- createdAt: "asc",
- },
- });
-
- const customerLeadEvents = await getLeadEvents({
- customerIds: customersData.map((customer) => customer.id),
- }).then((res) => res.data);
-
- await Promise.allSettled(
- commissions.map((commission) =>
- createCommission({
- program,
- commission,
- fxRates,
- importId,
- customersData,
- customerLeadEvents,
- }),
- ),
- );
-
- await new Promise((resolve) => setTimeout(resolve, 1000));
- }
- }
-
- if (!hasMore) {
- await partnerStackImporter.deleteCredentials(program.workspaceId);
- }
+ await new Promise((resolve) => setTimeout(resolve, 1000));
await partnerStackImporter.queue({
...payload,
startingAfter: hasMore ? currentStartingAfter : undefined,
- action: hasMore ? "import-commissions" : "update-stripe-customers",
+ action: hasMore ? "import-commissions" : "import-scheduled-commissions",
});
}
-async function createCommission({
+export async function createCommissionFromPS({
program,
commission,
fxRates,
diff --git a/apps/web/lib/partnerstack/import-scheduled-commissions.ts b/apps/web/lib/partnerstack/import-scheduled-commissions.ts
new file mode 100644
index 0000000000..bbe72144da
--- /dev/null
+++ b/apps/web/lib/partnerstack/import-scheduled-commissions.ts
@@ -0,0 +1,117 @@
+import { prisma } from "@dub/prisma";
+import { getLeadEvents } from "../tinybird/get-lead-events";
+import { redis } from "../upstash";
+import { PartnerStackApi } from "./api";
+import { createCommissionFromPS } from "./import-commissions";
+import { MAX_BATCHES, partnerStackImporter } from "./importer";
+import { PartnerStackImportPayload } from "./types";
+
+export async function importScheduledCommissions(
+ payload: PartnerStackImportPayload,
+) {
+ const { importId, programId, startingAfter } = payload;
+
+ const program = await prisma.program.findUniqueOrThrow({
+ where: {
+ id: programId,
+ },
+ });
+
+ const { publicKey, secretKey } = await partnerStackImporter.getCredentials(
+ program.workspaceId,
+ );
+
+ const partnerStackApi = new PartnerStackApi({
+ publicKey,
+ secretKey,
+ });
+
+ const fxRates = await redis.hgetall<Record<string, string>>("fxRates:usd");
+
+ let hasMore = true;
+ let processedBatches = 0;
+ let currentStartingAfter = startingAfter;
+
+ while (hasMore && processedBatches < MAX_BATCHES) {
+ const scheduledCommissions = await partnerStackApi.listCommissions({
+ status: "scheduled",
+ startingAfter: currentStartingAfter,
+ });
+
+ if (scheduledCommissions.length === 0) {
+ hasMore = false;
+ break;
+ }
+
+ const customersData = await prisma.customer.findMany({
+ where: {
+ projectId: program.workspaceId,
+ OR: [
+ {
+ email: {
+ in: scheduledCommissions
+ .map((commission) => commission.customer?.email)
+ .filter(
+ (email): email is string =>
+ email !== null && email !== undefined,
+ ),
+ },
+ },
+ {
+ externalId: {
+ in: scheduledCommissions
+ .map((commission) => commission.customer?.external_key)
+ .filter(
+ (externalKey): externalKey is string =>
+ externalKey !== null && externalKey !== undefined,
+ ),
+ },
+ },
+ ],
+ },
+ include: {
+ link: true,
+ },
+ orderBy: {
+ createdAt: "asc",
+ },
+ });
+
+ const customerLeadEvents = await getLeadEvents({
+ customerIds: customersData.map((customer) => customer.id),
+ }).then((res) => res.data);
+
+ await Promise.allSettled(
+ scheduledCommissions.map((commission) =>
+ createCommissionFromPS({
+ program,
+ commission,
+ fxRates,
+ importId,
+ customersData,
+ customerLeadEvents,
+ }),
+ ),
+ );
+
+ currentStartingAfter =
+ scheduledCommissions[scheduledCommissions.length - 1].key;
+ processedBatches++;
+ }
+
+ // If there are more scheduled commissions to import, sleep for 1 second and continue the loop
+ if (hasMore) {
+ await new Promise((resolve) => setTimeout(resolve, 1000));
+ } else {
+ // else, delete the credentials and exit the loop
+ await partnerStackImporter.deleteCredentials(program.workspaceId);
+ }
+
+ await partnerStackImporter.queue({
+ ...payload,
+ startingAfter: hasMore ? currentStartingAfter : undefined,
+ action: hasMore
+ ? "import-scheduled-commissions"
+ : "update-stripe-customers",
+ });
+}
diff --git a/apps/web/lib/partnerstack/schemas.ts b/apps/web/lib/partnerstack/schemas.ts
index dd1a253c95..13872553cf 100644
--- a/apps/web/lib/partnerstack/schemas.ts
+++ b/apps/web/lib/partnerstack/schemas.ts
@@ -6,6 +6,7 @@ export const partnerStackImportSteps = z.enum([
"import-links",
"import-customers",
"import-commissions",
+ "import-scheduled-commissions",
"update-stripe-customers",
]);
diff --git a/apps/web/lib/rewardful/api.ts b/apps/web/lib/rewardful/api.ts
index 6335e85800..e919b3cd36 100644
--- a/apps/web/lib/rewardful/api.ts
+++ b/apps/web/lib/rewardful/api.ts
@@ -7,7 +7,7 @@ import {
RewardfulReferral,
} from "./types";
-const PAGE_LIMIT = 100;
+const REWARDFUL_PAGE_LIMIT = 100;
class RewardfulApiError extends DubApiError {
constructor(message: string) {
@@ -57,7 +57,7 @@ export class RewardfulApi {
searchParams.append("expand[]", "campaign");
searchParams.append("expand[]", "links");
searchParams.append("page", page.toString());
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());
const { data } = await this.fetch<{ data: RewardfulAffiliate[] }>(
`${this.baseUrl}/affiliates?${searchParams.toString()}`,
@@ -72,7 +72,7 @@ export class RewardfulApi {
searchParams.append("conversion_state[]", "lead");
searchParams.append("conversion_state[]", "conversion");
searchParams.append("page", page.toString());
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());
const { data } = await this.fetch<{ data: RewardfulReferral[] }>(
`${this.baseUrl}/referrals?${searchParams.toString()}`,
@@ -86,7 +86,7 @@ export class RewardfulApi {
searchParams.append("expand[]", "sale");
searchParams.append("expand[]", "campaign");
searchParams.append("page", page.toString());
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());
const { data } = await this.fetch<{ data: RewardfulCommission[] }>(
`${this.baseUrl}/commissions?${searchParams.toString()}`,
@@ -98,7 +98,7 @@ export class RewardfulApi {
async listAffiliateCoupons({ page = 1 }: { page?: number }) {
const searchParams = new URLSearchParams();
searchParams.append("page", page.toString());
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", REWARDFUL_PAGE_LIMIT.toString());
const { data } = await this.fetch<{ data: RewardfulCoupon[] }>(
`${this.baseUrl}/affiliate_coupons?${searchParams.toString()}`,
diff --git a/apps/web/lib/tolt/api.ts b/apps/web/lib/tolt/api.ts
index 8f39a9ccbe..9748104d96 100644
--- a/apps/web/lib/tolt/api.ts
+++ b/apps/web/lib/tolt/api.ts
@@ -13,7 +13,7 @@ import {
ToltListResponse,
} from "./types";
-const PAGE_LIMIT = 100;
+const TOLT_PAGE_LIMIT = 100;
export class ToltApi {
private readonly baseUrl = "https://api.tolt.com/v1";
@@ -80,7 +80,7 @@ export class ToltApi {
const searchParams = new URLSearchParams();
searchParams.append("program_id", programId);
searchParams.append("expand[]", "program");
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());
if (startingAfter) {
searchParams.append("starting_after", startingAfter);
@@ -103,7 +103,7 @@ export class ToltApi {
const searchParams = new URLSearchParams();
searchParams.append("program_id", programId);
searchParams.append("expand[]", "partner");
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());
if (startingAfter) {
searchParams.append("starting_after", startingAfter);
@@ -126,7 +126,7 @@ export class ToltApi {
const searchParams = new URLSearchParams();
searchParams.append("program_id", programId);
searchParams.append("expand[]", "partner");
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());
if (startingAfter) {
searchParams.append("starting_after", startingAfter);
@@ -152,7 +152,7 @@ export class ToltApi {
searchParams.append("expand[]", "partner");
searchParams.append("expand[]", "customer");
searchParams.append("expand[]", "transaction");
- searchParams.append("limit", PAGE_LIMIT.toString());
+ searchParams.append("limit", TOLT_PAGE_LIMIT.toString());
if (startingAfter) {
searchParams.append("starting_after", startingAfter);