Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Sync partner social statistics (#3328)

Kiran K19ee424d8482
diff --git a/apps/web/app/(ee)/api/cron/partner-platforms/route.ts b/apps/web/app/(ee)/api/cron/partner-platforms/route.ts
new file mode 100644
index 0000000000..51314ea2e0
--- /dev/null
+++ b/apps/web/app/(ee)/api/cron/partner-platforms/route.ts
@@ -0,0 +1,156 @@
+import {
+ AccountNotFoundError,
+ fetchSocialProfile,
+} from "@/lib/api/scrape-creators/fetch-social-profile";
+import { qstash } from "@/lib/cron";
+import { withCron } from "@/lib/cron/with-cron";
+import { prisma } from "@dub/prisma";
+import { APP_DOMAIN_WITH_NGROK } from "@dub/utils";
+import { subDays } from "date-fns";
+import * as z from "zod/v4";
+import { logAndRespond } from "../utils";
+
+export const dynamic = "force-dynamic";
+
+const BATCH_SIZE = 50;
+
+const schema = z.object({
+ startingAfter: z.string().optional(),
+});
+
+/**
+ * This route is used to update stats for verified Instagram, TikTok, and Twitter partners using the ScrapeCreators API
+ * Runs once a day at 06:00 AM UTC (cron expression: 0 6 * * *)
+ * POST /api/cron/partner-platforms
+ */
+export const POST = withCron(async ({ rawBody }) => {
+ if (!process.env.SCRAPECREATORS_API_KEY) {
+ throw new Error("SCRAPECREATORS_API_KEY is not defined");
+ }
+
+ let { startingAfter } = schema.parse(
+ rawBody ? JSON.parse(rawBody) : { startingAfter: undefined },
+ );
+
+ const verifiedProfiles = await prisma.partnerPlatform.findMany({
+ where: {
+ type: {
+ in: ["instagram", "tiktok", "twitter"],
+ },
+ verifiedAt: {
+ not: null,
+ },
+ // only check platforms that haven't been checked in the last 7 days
+ OR: [
+ {
+ lastCheckedAt: {
+ lt: subDays(new Date(), 7),
+ },
+ },
+ {
+ lastCheckedAt: null,
+ },
+ ],
+ // only check partners that are discoverable in the partner network
+ partner: {
+ discoverableAt: {
+ not: null,
+ },
+ },
+ },
+ take: BATCH_SIZE,
+ ...(startingAfter && {
+ cursor: {
+ id: startingAfter,
+ },
+ skip: 1,
+ }),
+ orderBy: {
+ id: "asc",
+ },
+ });
+
+ if (verifiedProfiles.length === 0) {
+ return logAndRespond(
+ "No more verified social profiles found. Finished updating social platform stats.",
+ );
+ }
+
+ await Promise.allSettled(
+ verifiedProfiles.map(async (verifiedProfile) => {
+ if (!verifiedProfile.identifier || !verifiedProfile.type) {
+ return;
+ }
+
+ try {
+ const socialProfile = await fetchSocialProfile({
+ platform: verifiedProfile.type,
+ handle: verifiedProfile.identifier,
+ });
+
+ const newStats = {
+ subscribers: socialProfile.subscribers,
+ posts: socialProfile.posts,
+ };
+
+ await prisma.partnerPlatform.update({
+ where: {
+ id: verifiedProfile.id,
+ },
+ data: {
+ ...newStats,
+ lastCheckedAt: new Date(),
+ },
+ });
+
+ console.log(
+ `Updated ${verifiedProfile.type} stats for @${verifiedProfile.identifier}`,
+ newStats,
+ );
+ } catch (error) {
+ // If account doesn't exist, unverify the platform
+ if (error instanceof AccountNotFoundError) {
+ await prisma.partnerPlatform.update({
+ where: {
+ id: verifiedProfile.id,
+ },
+ data: {
+ verifiedAt: null,
+ lastCheckedAt: new Date(),
+ },
+ });
+
+ console.log(
+ `Account @${verifiedProfile.identifier} on ${verifiedProfile.type} no longer exists. Unverified platform.`,
+ );
+ return;
+ }
+
+ console.error(
+ `Error updating ${verifiedProfile.type} stats for @${verifiedProfile.identifier}:`,
+ error,
+ );
+ }
+ }),
+ );
+
+ if (verifiedProfiles.length === BATCH_SIZE) {
+ startingAfter = verifiedProfiles[verifiedProfiles.length - 1].id;
+
+ await qstash.publishJSON({
+ url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-platforms`,
+ method: "POST",
+ body: {
+ startingAfter,
+ },
+ });
+
+ return logAndRespond(
+ `Processed ${BATCH_SIZE} profiles. Scheduled next batch (startingAfter: ${startingAfter}).`,
+ );
+ }
+
+ return logAndRespond(
+ "Finished updating social platform stats for all verified profiles.",
+ );
+});
diff --git a/apps/web/app/(ee)/api/cron/online-presence/youtube/route.ts b/apps/web/app/(ee)/api/cron/partner-platforms/youtube/route.ts
similarity index 87%
rename from apps/web/app/(ee)/api/cron/online-presence/youtube/route.ts
rename to apps/web/app/(ee)/api/cron/partner-platforms/youtube/route.ts
index 25692a17f7..b1f731f88d 100644
--- a/apps/web/app/(ee)/api/cron/online-presence/youtube/route.ts
+++ b/apps/web/app/(ee)/api/cron/partner-platforms/youtube/route.ts
@@ -1,8 +1,8 @@
import { withCron } from "@/lib/cron/with-cron";
import { prisma } from "@dub/prisma";
import { PlatformType } from "@dub/prisma/client";
-import { chunk, deepEqual } from "@dub/utils";
-import { z } from "zod";
+import { chunk } from "@dub/utils";
+import * as z from "zod/v4";
import { logAndRespond } from "../../utils";
const youtubeChannelSchema = z.object({
@@ -16,10 +16,11 @@ const youtubeChannelSchema = z.object({
export const dynamic = "force-dynamic";
-/*
- This route is used to update youtube stats for youtubeVerified partners
- Runs once a day at 06:00 AM UTC (cron expression: 0 6 * * *)
-*/
+/**
+ * This route is used to update stats for YouTube verified partners using the YouTube API
+ * Runs once a day at 06:00 AM UTC (cron expression: 0 6 * * *)
+ * POST /api/cron/partner-platforms/youtube
+ */
export const POST = withCron(async () => {
if (!process.env.YOUTUBE_API_KEY) {
throw new Error("YOUTUBE_API_KEY is not defined");
@@ -88,15 +89,14 @@ export const POST = withCron(async () => {
views: channel.statistics.viewCount,
};
- if (deepEqual(currentStats, newStats)) {
- continue;
- }
-
await prisma.partnerPlatform.update({
where: {
id: partnerPlatform.id,
},
- data: newStats,
+ data: {
+ ...newStats,
+ lastCheckedAt: new Date(),
+ },
});
console.log(
diff --git a/apps/web/app/(ee)/api/partners/online-presence/callback/route.ts b/apps/web/app/(ee)/api/partners/online-presence/callback/route.ts
index a875f1f74a..162139d299 100644
--- a/apps/web/app/(ee)/api/partners/online-presence/callback/route.ts
+++ b/apps/web/app/(ee)/api/partners/online-presence/callback/route.ts
@@ -1,8 +1,9 @@
import { ONLINE_PRESENCE_PROVIDERS } from "@/lib/api/partner-profile/online-presence-providers";
+import { fetchSocialProfile } from "@/lib/api/scrape-creators/fetch-social-profile";
import { getSession } from "@/lib/auth/utils";
import { redis } from "@/lib/upstash/redis";
import { prisma } from "@dub/prisma";
-import { PlatformType } from "@dub/prisma/client";
+import { PartnerPlatform, PlatformType } from "@dub/prisma/client";
import {
getSearchParams,
PARTNERS_DOMAIN,
@@ -10,7 +11,7 @@ import {
} from "@dub/utils";
import { cookies } from "next/headers";
import { NextResponse } from "next/server";
-import { z } from "zod";
+import * as z from "zod/v4";
const requestSchema = z.object({
code: z.string(),
@@ -148,6 +149,33 @@ export async function GET(req: Request) {
return NextResponse.redirect(redirectUrl);
}
+ // Sync social stats for platforms
+ let socialStats: Pick<PartnerPlatform, "subscribers" | "posts" | "views"> = {
+ subscribers: BigInt(0),
+ posts: BigInt(0),
+ views: BigInt(0),
+ };
+
+ if (["tiktok", "twitter"].includes(platform)) {
+ try {
+ const socialProfile = await fetchSocialProfile({
+ platform,
+ handle: partnerPlatform.identifier,
+ });
+
+ socialStats = {
+ subscribers: socialProfile.subscribers,
+ posts: socialProfile.posts,
+ views: socialProfile.views,
+ };
+ } catch (error) {
+ console.error(
+ `Failed to fetch social stats for ${platform} handle @${partnerPlatform.identifier}:`,
+ error,
+ );
+ }
+ }
+
await prisma.partnerPlatform.update({
where: {
partnerId_type: {
@@ -157,7 +185,8 @@ export async function GET(req: Request) {
},
data: {
verifiedAt: new Date(),
- metadata: metadata || undefined,
+ ...(metadata && { metadata }),
+ ...socialStats,
},
});
diff --git a/apps/web/app/(ee)/partners.dub.co/(onboarding)/onboarding/online-presence/page.tsx b/apps/web/app/(ee)/partners.dub.co/(onboarding)/onboarding/online-presence/page.tsx
index 0a76102960..eae8a64a6a 100644
--- a/apps/web/app/(ee)/partners.dub.co/(onboarding)/onboarding/online-presence/page.tsx
+++ b/apps/web/app/(ee)/partners.dub.co/(onboarding)/onboarding/online-presence/page.tsx
@@ -6,7 +6,7 @@ import { OnlinePresenceForm } from "@/ui/partners/online-presence-form";
import { prisma } from "@dub/prisma";
import { PlatformType } from "@dub/prisma/client";
import { Suspense } from "react";
-import { z } from "zod";
+import * as z from "zod/v4";
import { OnlinePresencePageClient } from "./page-client";
export default function OnlinePresencePage() {
diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx
index 9f2f39c186..e5f951e24d 100644
--- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx
+++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx
@@ -1,7 +1,7 @@
"use client";
import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner";
-import { ONLINE_PRESENCE_FIELDS } from "@/lib/partners/online-presence";
+import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms";
import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count";
import useWorkspace from "@/lib/swr/use-workspace";
import { NetworkPartnerProps } from "@/lib/types";
@@ -383,7 +383,7 @@ function PartnerCard({
const onlinePresenceData = useMemo(
() =>
partner
- ? ONLINE_PRESENCE_FIELDS.map((field) => ({
+ ? PARTNER_PLATFORM_FIELDS.map((field) => ({
label: field.label,
icon: field.icon,
...field.data(partner.platforms),
diff --git a/apps/web/lib/actions/partners/start-partner-platform-verification.ts b/apps/web/lib/actions/partners/start-partner-platform-verification.ts
index 056da5761f..9f553b4fce 100644
--- a/apps/web/lib/actions/partners/start-partner-platform-verification.ts
+++ b/apps/web/lib/actions/partners/start-partner-platform-verification.ts
@@ -17,7 +17,7 @@ import { PlatformType } from "@dub/prisma/client";
import { nanoid, PARTNERS_DOMAIN_WITH_NGROK } from "@dub/utils";
import { cookies } from "next/headers";
import { v4 as uuid } from "uuid";
-import { z } from "zod";
+import * as z from "zod/v4";
import { authPartnerActionClient } from "../safe-action";
const startPartnerPlatformVerificationSchema = z.object({
diff --git a/apps/web/lib/actions/partners/verify-social-account-by-code.ts b/apps/web/lib/actions/partners/verify-social-account-by-code.ts
index 933e2b285d..d16a704dd0 100644
--- a/apps/web/lib/actions/partners/verify-social-account-by-code.ts
+++ b/apps/web/lib/actions/partners/verify-social-account-by-code.ts
@@ -1,6 +1,6 @@
"use server";
-import { scrapeCreatorsClient } from "@/lib/api/scrapecreators/client";
+import { fetchSocialProfile } from "@/lib/api/scrape-creators/fetch-social-profile";
import { ratelimit } from "@/lib/upstash";
import { redis } from "@/lib/upstash/redis";
import { prisma } from "@dub/prisma";
@@ -74,18 +74,12 @@ export const verifySocialAccountByCodeAction = authPartnerActionClient
// Verifies that a verification code exists in the account's profile bio/description.
// Fetches the account profile and checks if the provided code appears in any of the
// profile text fields (description, about, bio, summary).
- const socialProfile = await scrapeCreatorsClient.fetchSocialProfile({
+ const socialProfile = await fetchSocialProfile({
platform,
handle,
});
- if (!socialProfile) {
- throw new Error(
- "We were unable to retrieve your social media profile. Please try again.",
- );
- }
-
- if (!socialProfile.description) {
+ if (!socialProfile.description || socialProfile.description.length === 0) {
throw new Error(
`We could not find a public ${
platform === "youtube" ? "channel description" : "bio"
@@ -113,9 +107,11 @@ export const verifySocialAccountByCodeAction = authPartnerActionClient
data: {
verifiedAt: new Date(),
platformId: socialProfile.platformId,
+ subscribers: socialProfile.subscribers,
+ posts: socialProfile.posts,
+ views: socialProfile.views,
},
});
- // Delete the verification code from Redis
await redis.del(cacheKey);
});
diff --git a/apps/web/lib/api/partner-profile/online-presence-providers.ts b/apps/web/lib/api/partner-profile/online-presence-providers.ts
index 33e1d5b34f..9229745822 100644
--- a/apps/web/lib/api/partner-profile/online-presence-providers.ts
+++ b/apps/web/lib/api/partner-profile/online-presence-providers.ts
@@ -117,6 +117,7 @@ export const ONLINE_PRESENCE_PROVIDERS: Record<string, OnlinePresenceProvider> =
},
},
+ // We don't support LinkedIn verification yet
linkedin: {
authUrl: "https://www.linkedin.com/oauth/v2/authorization",
tokenUrl: "https://www.linkedin.com/oauth/v2/accessToken",
diff --git a/apps/web/lib/api/scrape-creators/client.ts b/apps/web/lib/api/scrape-creators/client.ts
new file mode 100644
index 0000000000..72cf0b75f2
--- /dev/null
+++ b/apps/web/lib/api/scrape-creators/client.ts
@@ -0,0 +1,43 @@
+import { createFetch, createSchema } from "@better-fetch/fetch";
+import { PlatformType } from "@dub/prisma/client";
+import * as z from "zod/v4";
+import { profileResponseSchema } from "./schema";
+
+export const scrapeCreatorsFetch = createFetch({
+ baseURL: "https://api.scrapecreators.com",
+ retry: {
+ type: "linear",
+ attempts: 2,
+ delay: 3000,
+ },
+ headers: {
+ "x-api-key": process.env.SCRAPECREATORS_API_KEY!,
+ },
+ schema: createSchema(
+ {
+ "/v1/:platform/:handleType": {
+ method: "get",
+ params: z.object({
+ platform: z.enum(PlatformType),
+ handleType: z.enum(["channel", "profile"]),
+ }),
+ query: z.object({
+ handle: z.string(),
+ }),
+ output: profileResponseSchema,
+ },
+ },
+ { strict: true },
+ ),
+ onError: ({ error }) => {
+ console.error("[ScrapeCreators] Error", error);
+ },
+ // onResponse: async ({ response }) => {
+ // if (process.env.NODE_ENV === "development") {
+ // console.log(
+ // "[ScrapeCreators] Response",
+ // prettyPrint(await response.clone().json()),
+ // );
+ // }
+ // },
+});
diff --git a/apps/web/lib/api/scrape-creators/fetch-social-profile.ts b/apps/web/lib/api/scrape-creators/fetch-social-profile.ts
new file mode 100644
index 0000000000..bcb42560eb
--- /dev/null
+++ b/apps/web/lib/api/scrape-creators/fetch-social-profile.ts
@@ -0,0 +1,98 @@
+import { PartnerPlatform, PlatformType } from "@dub/prisma/client";
+import { scrapeCreatorsFetch } from "./client";
+
+type FetchSocialProfileParams = {
+ platform: PlatformType;
+ handle: string;
+};
+
+type SocialProfile = Pick<
+ PartnerPlatform,
+ "platformId" | "subscribers" | "posts" | "views"
+> & {
+ description: string | null;
+};
+
+export class AccountNotFoundError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = "AccountNotFoundError";
+ }
+}
+
+export async function fetchSocialProfile({
+ platform,
+ handle,
+}: FetchSocialProfileParams) {
+ const { data, error } = await scrapeCreatorsFetch(
+ "/v1/:platform/:handleType",
+ {
+ params: {
+ platform,
+ handleType: platform === "youtube" ? "channel" : "profile",
+ },
+ query: {
+ handle,
+ },
+ },
+ );
+
+ if (error) {
+ throw new Error(
+ "We were unable to retrieve your social media profile. Please try again.",
+ );
+ }
+
+ // Check if account doesn't exist
+ if (data.platform === "account_not_found") {
+ throw new AccountNotFoundError(
+ (data as { message?: string }).message || "Account doesn't exist",
+ );
+ }
+
+ let socialProfile: SocialProfile = {
+ description: null,
+ platformId: null,
+ subscribers: BigInt(0),
+ posts: BigInt(0),
+ views: BigInt(0),
+ };
+
+ switch (data.platform) {
+ case "youtube": {
+ socialProfile.description = data.description;
+ socialProfile.platformId = data.channelId;
+ socialProfile.subscribers = BigInt(data.subscriberCount);
+ socialProfile.posts = BigInt(data.videoCount);
+ socialProfile.views = BigInt(data.viewCount);
+ break;
+ }
+
+ case "instagram": {
+ socialProfile.description = data.data.user.biography;
+ socialProfile.subscribers = BigInt(data.data.user.edge_followed_by.count);
+ socialProfile.posts = BigInt(
+ data.data.user.edge_owner_to_timeline_media.count,
+ );
+ break;
+ }
+
+ case "tiktok": {
+ socialProfile.description = data.user.signature;
+ socialProfile.platformId = data.user.id;
+ socialProfile.subscribers = BigInt(data.stats.followerCount);
+ socialProfile.posts = BigInt(data.stats.videoCount);
+ break;
+ }
+
+ case "twitter": {
+ socialProfile.description = data.legacy.description;
+ socialProfile.platformId = data.rest_id;
+ socialProfile.subscribers = BigInt(data.legacy.followers_count);
+ socialProfile.posts = BigInt(data.legacy.statuses_count);
+ break;
+ }
+ }
+
+ return socialProfile;
+}
diff --git a/apps/web/lib/api/scrape-creators/schema.ts b/apps/web/lib/api/scrape-creators/schema.ts
new file mode 100644
index 0000000000..58a5b58c95
--- /dev/null
+++ b/apps/web/lib/api/scrape-creators/schema.ts
@@ -0,0 +1,95 @@
+import * as z from "zod/v4";
+
+export const profileResponseSchema = z.preprocess(
+ (data: unknown) => {
+ if (typeof data === "object" && data !== null) {
+ // Check for "account doesn't exist" response
+ if (
+ "message" in data &&
+ typeof (data as any).message === "string" &&
+ ((data as any).message.toLowerCase().includes("doesn't exist") ||
+ (data as any).message.toLowerCase().includes("does not exist") ||
+ (data as any).message.toLowerCase().includes("not found"))
+ ) {
+ return { platform: "account_not_found", ...data };
+ }
+
+ if ("description" in data && "channelId" in data) {
+ return { platform: "youtube", ...data };
+ }
+
+ if (
+ "data" in data &&
+ typeof (data as any).data === "object" &&
+ (data as any).data?.user
+ ) {
+ return { platform: "instagram", ...data };
+ }
+
+ if ("user" in data && "stats" in data) {
+ return { platform: "tiktok", ...data };
+ }
+
+ // Twitter detection: check for rest_id and legacy fields (more reliable than is_blue_verified)
+ if ("rest_id" in data && "legacy" in data) {
+ return { platform: "twitter", ...data };
+ }
+ }
+
+ return data;
+ },
+
+ z.discriminatedUnion("platform", [
+ z.object({
+ platform: z.literal("account_not_found"),
+ handle: z.string().optional(),
+ message: z.string().optional(),
+ }),
+
+ z.object({
+ platform: z.literal("youtube"),
+ description: z.string(),
+ channelId: z.string(),
+ videoCount: z.number(),
+ subscriberCount: z.number(),
+ viewCount: z.number(),
+ }),
+
+ z.object({
+ platform: z.literal("instagram"),
+ data: z.object({
+ user: z.object({
+ biography: z.string(),
+ edge_followed_by: z.object({ count: z.number() }),
+ edge_owner_to_timeline_media: z.object({ count: z.number() }),
+ }),
+ }),
+ }),
+
+ z.object({
+ platform: z.literal("tiktok"),
+ user: z.object({
+ id: z.string(),
+ signature: z.string(),
+ uniqueId: z.string(),
+ }),
+ stats: z.object({
+ followerCount: z.number(),
+ videoCount: z.number(),
+ heartCount: z.number(),
+ }),
+ }),
+
+ z.object({
+ platform: z.literal("twitter"),
+ rest_id: z.string(),
+ legacy: z.object({
+ description: z.string(),
+ followers_count: z.number(),
+ statuses_count: z.number(),
+ }),
+ }),
+ ]),
+);
+
+export type ProfileResponse = z.infer<typeof profileResponseSchema>;
diff --git a/apps/web/lib/api/scrapecreators/client.ts b/apps/web/lib/api/scrapecreators/client.ts
deleted file mode 100644
index 0605b5b510..0000000000
--- a/apps/web/lib/api/scrapecreators/client.ts
+++ /dev/null
@@ -1,125 +0,0 @@
-import { PlatformType } from "@dub/prisma/client";
-import { z } from "zod";
-
-const SCRAPECREATORS_API_BASE_URL = "https://api.scrapecreators.com";
-const SCRAPECREATORS_API_KEY = process.env.SCRAPECREATORS_API_KEY || "";
-
-export type ProfileFields = {
- description: string;
- platformId: string | null; // Eg. YouTube channel Id
-};
-
-// Platform-specific response schemas
-const youtubeResponseSchema = z.object({
- description: z.string(),
- channelId: z.string(),
-});
-
-const instagramResponseSchema = z.object({
- data: z.object({
- user: z.object({
- biography: z.string(),
- }),
- }),
-});
-
-const errorResponseSchema = z.object({
- message: z.string(),
-});
-
-type PlatformRequestConfig = {
- path: string;
- buildSearchParams: (handle: string) => URLSearchParams;
- parseResponse: (data: unknown) => ProfileFields;
-};
-
-const PLATFORM_CONFIG: Record<
- Exclude<PlatformType, "linkedin" | "tiktok" | "twitter" | "website">,
- PlatformRequestConfig
-> = {
- youtube: {
- path: "/v1/youtube/channel",
- buildSearchParams: (handle) => new URLSearchParams({ handle }),
- parseResponse: (data) => {
- const parsed = youtubeResponseSchema.parse(data);
-
- return {
- description: parsed.description,
- platformId: parsed.channelId,
- };
- },
- },
- instagram: {
- path: "/v1/instagram/profile",
- buildSearchParams: (handle) => new URLSearchParams({ handle }),
- parseResponse: (data) => {
- const parsed = instagramResponseSchema.parse(data);
-
- return {
- description: parsed.data.user.biography,
- platformId: null,
- };
- },
- },
-};
-
-export class ScrapeCreatorsClient {
- private apiKey: string;
- private baseUrl: string;
-
- constructor() {
- if (!SCRAPECREATORS_API_KEY) {
- throw new Error("SCRAPECREATORS_API_KEY is not configured.");
- }
-
- this.apiKey = SCRAPECREATORS_API_KEY;
- this.baseUrl = SCRAPECREATORS_API_BASE_URL;
- }
-
- // Fetches public profile text data for a supported social platform
- public async fetchSocialProfile({
- platform,
- handle,
- }: {
- platform: PlatformType;
- handle: string;
- }): Promise<ProfileFields> {
- const config = PLATFORM_CONFIG[platform];
-
- if (!config) {
- throw new Error(`Unsupported platform: ${platform}`);
- }
-
- const endpoint = `${this.baseUrl}${config.path}`;
- const searchParams = config.buildSearchParams(handle);
-
- const response = await fetch(`${endpoint}?${searchParams.toString()}`, {
- headers: {
- "x-api-key": this.apiKey,
- },
- signal: AbortSignal.timeout(10000), // 10 second timeout
- });
-
- const jsonResponse = await response.json();
-
- if (!response.ok) {
- console.error("[ScrapeCreators] Failed to fetch account", {
- ...jsonResponse,
- platform,
- handle,
- });
-
- const parsedError = errorResponseSchema.safeParse(jsonResponse);
-
- throw new Error(
- parsedError.success
- ? parsedError.data.message
- : "Failed to fetch account profile.",
- );
- }
-
- return config.parseResponse(jsonResponse);
- }
-}
-
-export const scrapeCreatorsClient = new ScrapeCreatorsClient();
diff --git a/apps/web/lib/network/get-discoverability-requirements.ts b/apps/web/lib/network/get-discoverability-requirements.ts
index aaa4235aa4..2ecbdd8af2 100644
--- a/apps/web/lib/network/get-discoverability-requirements.ts
+++ b/apps/web/lib/network/get-discoverability-requirements.ts
@@ -3,7 +3,7 @@ import {
EXCLUDED_PROGRAM_IDS,
PARTNER_NETWORK_MIN_COMMISSIONS_CENTS,
} from "../constants/partner-profile";
-import { ONLINE_PRESENCE_FIELDS } from "../partners/online-presence";
+import { PARTNER_PLATFORM_FIELDS } from "../partners/partner-platforms";
import { EnrolledPartnerProps, PartnerProps } from "../types";
export const partnerHasEarnedCommissions = (
@@ -72,7 +72,7 @@ export function getDiscoverabilityRequirements({
{
label: "Connect your website or social account",
href: "#sites",
- completed: ONLINE_PRESENCE_FIELDS.some(
+ completed: PARTNER_PLATFORM_FIELDS.some(
(field) => field.data(partner.platforms).value, // TODO: update this to also check for "verified" in the future
),
},
diff --git a/apps/web/lib/partners/online-presence.ts b/apps/web/lib/partners/partner-platforms.ts
similarity index 75%
rename from apps/web/lib/partners/online-presence.ts
rename to apps/web/lib/partners/partner-platforms.ts
index e92c1a107a..44e1702664 100644
--- a/apps/web/lib/partners/online-presence.ts
+++ b/apps/web/lib/partners/partner-platforms.ts
@@ -10,7 +10,7 @@ import {
import { getPrettyUrl, nFormatter } from "@dub/utils";
import { PartnerPlatformProps } from "../types";
-export const ONLINE_PRESENCE_FIELDS: {
+export const PARTNER_PLATFORM_FIELDS: {
label: string;
icon: Icon;
data: (platforms: PartnerPlatformProps[]) => {
@@ -68,6 +68,14 @@ export const ONLINE_PRESENCE_FIELDS: {
href: twitter?.identifier
? `https://x.com/${twitter.identifier}`
: null,
+ info: [
+ twitter?.subscribers && twitter.subscribers > 0
+ ? `${nFormatter(Number(twitter.subscribers))} followers`
+ : null,
+ twitter?.posts && twitter.posts > 0
+ ? `${nFormatter(Number(twitter.posts))} tweets`
+ : null,
+ ].filter(Boolean),
};
},
},
@@ -98,6 +106,14 @@ export const ONLINE_PRESENCE_FIELDS: {
href: instagram?.identifier
? `https://instagram.com/${instagram.identifier}`
: null,
+ info: [
+ instagram?.subscribers && instagram.subscribers > 0
+ ? `${nFormatter(Number(instagram.subscribers))} followers`
+ : null,
+ instagram?.posts && instagram.posts > 0
+ ? `${nFormatter(Number(instagram.posts))} posts`
+ : null,
+ ].filter(Boolean),
};
},
},
@@ -113,6 +129,14 @@ export const ONLINE_PRESENCE_FIELDS: {
href: tiktok?.identifier
? `https://tiktok.com/@${tiktok.identifier}`
: null,
+ info: [
+ tiktok?.subscribers && tiktok.subscribers > 0
+ ? `${nFormatter(Number(tiktok.subscribers))} followers`
+ : null,
+ tiktok?.posts && tiktok.posts > 0
+ ? `${nFormatter(Number(tiktok.posts))} posts`
+ : null,
+ ].filter(Boolean),
};
},
},
diff --git a/apps/web/package.json b/apps/web/package.json
index f42734d868..df32e16b39 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -22,6 +22,7 @@
"@axiomhq/js": "^1.3.1",
"@axiomhq/logging": "^0.1.5",
"@axiomhq/nextjs": "^0.1.6",
+ "@better-fetch/fetch": "^1.1.21",
"@boxyhq/saml-jackson": "1.52.1",
"@chronark/zod-bird": "^1.0.0",
"@date-fns/tz": "^1.4.1",
diff --git a/apps/web/ui/partners/online-presence-form.tsx b/apps/web/ui/partners/online-presence-form.tsx
index d1699877c5..2496c627d4 100644
--- a/apps/web/ui/partners/online-presence-form.tsx
+++ b/apps/web/ui/partners/online-presence-form.tsx
@@ -661,24 +661,75 @@ function FormRow({
const { isVerified } = useVerifiedState({ property });
const info = useMemo(() => {
- if (partner && property === "youtube" && isVerified) {
+ if (partner && isVerified) {
// Type assertion for platforms array that exists at runtime but not in type
const partnerWithPlatforms = partner as typeof partner & {
platforms?: PartnerPlatformProps[];
};
- const youtubePlatform = getPlatformData(
- partnerWithPlatforms.platforms,
- "youtube",
- );
- const subscribers = youtubePlatform?.subscribers ?? 0;
- const views = youtubePlatform?.views ?? 0;
-
- return [
- subscribers > 0
- ? `${nFormatter(Number(subscribers))} subscribers`
- : null,
- views > 0 ? `${nFormatter(Number(views))} views` : null,
- ].filter(Boolean) as string[];
+
+ if (property === "youtube") {
+ const youtubePlatform = getPlatformData(
+ partnerWithPlatforms.platforms,
+ "youtube",
+ );
+ const subscribers = youtubePlatform?.subscribers ?? 0;
+ const views = youtubePlatform?.views ?? 0;
+
+ return [
+ subscribers > 0
+ ? `${nFormatter(Number(subscribers))} subscribers`
+ : null,
+ views > 0 ? `${nFormatter(Number(views))} views` : null,
+ ].filter(Boolean) as string[];
+ }
+
+ if (property === "instagram") {
+ const instagramPlatform = getPlatformData(
+ partnerWithPlatforms.platforms,
+ "instagram",
+ );
+ const subscribers = instagramPlatform?.subscribers ?? 0;
+ const posts = instagramPlatform?.posts ?? 0;
+
+ return [
+ subscribers > 0
+ ? `${nFormatter(Number(subscribers))} followers`
+ : null,
+ posts > 0 ? `${nFormatter(Number(posts))} posts` : null,
+ ].filter(Boolean) as string[];
+ }
+
+ if (property === "tiktok") {
+ const tiktokPlatform = getPlatformData(
+ partnerWithPlatforms.platforms,
+ "tiktok",
+ );
+ const subscribers = tiktokPlatform?.subscribers ?? 0;
+ const posts = tiktokPlatform?.posts ?? 0;
+
+ return [
+ subscribers > 0
+ ? `${nFormatter(Number(subscribers))} followers`
+ : null,
+ posts > 0 ? `${nFormatter(Number(posts))} posts` : null,
+ ].filter(Boolean) as string[];
+ }
+
+ if (property === "twitter") {
+ const twitterPlatform = getPlatformData(
+ partnerWithPlatforms.platforms,
+ "twitter",
+ );
+ const subscribers = twitterPlatform?.subscribers ?? 0;
+ const posts = twitterPlatform?.posts ?? 0;
+
+ return [
+ subscribers > 0
+ ? `${nFormatter(Number(subscribers))} followers`
+ : null,
+ posts > 0 ? `${nFormatter(Number(posts))} tweets` : null,
+ ].filter(Boolean) as string[];
+ }
}
return null;
}, [partner, property, isVerified]);
diff --git a/apps/web/ui/partners/online-presence-summary.tsx b/apps/web/ui/partners/online-presence-summary.tsx
index c84eb321b0..f065352fcf 100644
--- a/apps/web/ui/partners/online-presence-summary.tsx
+++ b/apps/web/ui/partners/online-presence-summary.tsx
@@ -1,4 +1,4 @@
-import { ONLINE_PRESENCE_FIELDS } from "@/lib/partners/online-presence";
+import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms";
import { PartnerPlatformProps } from "@/lib/types";
import { cn } from "@dub/utils";
import { Fragment } from "react";
@@ -29,7 +29,7 @@ export function OnlinePresenceSummary({
);
}
- const fieldData = ONLINE_PRESENCE_FIELDS.map((field) => ({
+ const fieldData = PARTNER_PLATFORM_FIELDS.map((field) => ({
label: field.label,
icon: field.icon,
...field.data(platforms),
diff --git a/apps/web/vercel.json b/apps/web/vercel.json
index aaf11ac447..2293e47fa4 100644
--- a/apps/web/vercel.json
+++ b/apps/web/vercel.json
@@ -36,6 +36,10 @@
"path": "/api/cron/partner-program-summary",
"schedule": "0 13 1 * *"
},
+ {
+ "path": "/api/cron/pending-applications-summary",
+ "schedule": "0 9 * * *"
+ },
{
"path": "/api/cron/payouts/aggregate-due-commissions",
"schedule": "0 * * * *"
@@ -47,10 +51,6 @@
{
"path": "/api/cron/payouts/reminders/program-owners",
"schedule": "0 13 25-31,1-5 * *"
- },
- {
- "path": "/api/cron/pending-applications-summary",
- "schedule": "0 9 * * *"
}
],
"functions": {
diff --git a/packages/prisma/schema/platform.prisma b/packages/prisma/schema/platform.prisma
index 379839b440..e9f56d0430 100644
--- a/packages/prisma/schema/platform.prisma
+++ b/packages/prisma/schema/platform.prisma
@@ -8,18 +8,19 @@ enum PlatformType {
}
model PartnerPlatform {
- id String @id @default(cuid())
- partnerId String
- type PlatformType
- identifier String // The unique identifier for the platform (e.g., username for social platforms, domain for websites)
- platformId String? // Platform-specific immutable ID (e.g., YouTube channel ID)
- verifiedAt DateTime?
- subscribers BigInt @default(0) // Subscribers/followers for social platforms, also for newsletter subscribers
- posts BigInt @default(0)
- views BigInt @default(0) // Video views for YouTube, impressions for social platforms, pageviews for websites
- metadata Json?
- createdAt DateTime @default(now())
- updatedAt DateTime @updatedAt
+ id String @id @default(cuid())
+ partnerId String
+ type PlatformType
+ identifier String // The unique identifier for the platform (e.g., username for social platforms, domain for websites)
+ platformId String? // Platform-specific immutable ID (e.g., YouTube channel ID)
+ subscribers BigInt @default(0) // Subscribers/followers for social platforms, also for newsletter subscribers
+ posts BigInt @default(0)
+ views BigInt @default(0) // Video views for YouTube, impressions for social platforms, pageviews for websites
+ metadata Json?
+ createdAt DateTime @default(now())
+ updatedAt DateTime @updatedAt
+ verifiedAt DateTime?
+ lastCheckedAt DateTime?
partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade)
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d7669af89a..2e4132ecc0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -53,6 +53,9 @@ importers:
'@axiomhq/nextjs':
specifier: ^0.1.6
+ '@better-fetch/fetch':
+ specifier: ^1.1.21
+ version: 1.1.21
'@boxyhq/saml-jackson':
specifier: 1.52.1
@@ -642,7 +645,7 @@ importers:
dependencies:
'@hubspot/cli':
specifier: ^7.6.2
devDependencies:
'@types/node':
specifier: 18.11.9
@@ -1572,6 +1575,9 @@ packages:
resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==}
+ '@better-fetch/[email protected]':
+ resolution: {integrity: sha512-/ImESw0sskqlVR94jB+5+Pxjf+xBwDZF/N5+y2/q4EqD7IARUTSpPfIo8uf39SYpCxyOCtbyYpUrZ3F/k0zT4A==}
+
'@boxyhq/[email protected]':
resolution: {integrity: sha512-NmO111OG8GQDE8W/+uSREb67YSqnY2N/tHykGeFoIZc9Leher+lW+jN4U1OXzlc66hwB8yO7WRu2cbYsAKsi9g==}
@@ -13075,10 +13081,12 @@ packages:
resolution: {integrity: sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==}
+ deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==}
engines: {node: '>=18'}
+ deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
resolution: {integrity: sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==}
@@ -14281,9 +14289,9 @@ snapshots:
'@babel/core': 7.24.5
'@babel/helper-plugin-utils': 7.25.9
dependencies:
- '@babel/core': 7.28.4
+ '@babel/core': 7.24.5
'@babel/helper-plugin-utils': 7.27.1
optional: true
@@ -14400,6 +14408,8 @@ snapshots:
'@bcoe/[email protected]': {}
+ '@better-fetch/[email protected]': {}
+
'@boxyhq/[email protected]': {}
'@boxyhq/[email protected]':
@@ -15033,7 +15043,7 @@ snapshots:
- encoding
- supports-color
dependencies:
'@hubspot/local-dev-lib': 3.19.1
'@hubspot/project-parsing-lib': 0.8.6(@hubspot/[email protected])
@@ -15064,7 +15074,7 @@ snapshots:
yargs: 17.7.2
yargs-parser: 21.1.1
optionalDependencies:
'@modelcontextprotocol/sdk': 1.13.3
transitivePeerDependencies:
- '@babel/core'
@@ -15102,7 +15112,7 @@ snapshots:
react-dom: 18.3.1([email protected])
optional: true
dependencies:
'@babel/code-frame': 7.26.2
'@babel/parser': 7.26.2
@@ -15129,7 +15139,7 @@ snapshots:
'@vitejs/plugin-react': 4.7.0([email protected](@types/[email protected])([email protected]))
ansi-to-html: 0.7.2
babel-plugin-macros: 3.1.0
chalk: 5.4.1
class-variance-authority: 0.7.0
cli-progress: 3.12.0
@@ -15148,7 +15158,7 @@ snapshots:
react-dom: 18.3.1([email protected])
request: 2.88.2
storybook: 8.6.14([email protected])
tailwind-merge: 2.6.0
tailwindcss-animate: 1.0.7
typescript: 4.7.4
@@ -20160,11 +20170,11 @@ snapshots:
resolve: 1.22.6
optional: true
dependencies:
'@babel/helper-annotate-as-pure': 7.27.3
'@babel/helper-module-imports': 7.25.9
- '@babel/plugin-syntax-jsx': 7.27.1(@babel/[email protected])
+ '@babel/plugin-syntax-jsx': 7.27.1(@babel/[email protected])
lodash: 4.17.21
picomatch: 2.3.1
@@ -27078,12 +27088,12 @@ snapshots:
tslib: 2.6.2
optional: true
dependencies:
client-only: 0.0.1
react: 18.3.1
optionalDependencies:
- '@babel/core': 7.28.4
+ '@babel/core': 7.24.5
babel-plugin-macros: 3.1.0
optional: true