Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

update partner-platforms/website cron

Steven Teyeab1b8295da7
diff --git a/apps/web/app/(ee)/api/admin/partners/[partnerId]/shared-platforms/route.ts b/apps/web/app/(ee)/api/admin/partners/[partnerId]/shared-platforms/route.ts
index 65f3c9958d..282e408a5c 100644
--- a/apps/web/app/(ee)/api/admin/partners/[partnerId]/shared-platforms/route.ts
+++ b/apps/web/app/(ee)/api/admin/partners/[partnerId]/shared-platforms/route.ts
@@ -41,18 +41,18 @@ export const GET = withAdmin(async ({ params }) => {
// so match on the domain instead of the exact identifier
if (websiteDomain) {
matchConditions.push({
- type: platform.type,
identifier: {
contains: websiteDomain,
},
+ type: platform.type,
});
}
continue;
}
matchConditions.push({
- type: platform.type,
identifier: platform.identifier,
+ type: platform.type,
});
}
@@ -62,13 +62,13 @@ export const GET = withAdmin(async ({ params }) => {
const sharedPlatformMatches = await prisma.partnerPlatform.findMany({
where: {
+ OR: matchConditions,
partnerId: {
not: partner.id,
},
verifiedAt: {
not: null,
},
- OR: matchConditions,
},
include: {
partner: {
diff --git a/apps/web/app/(ee)/api/cron/partner-platforms/website/route.ts b/apps/web/app/(ee)/api/cron/partner-platforms/website/route.ts
index ada3c15449..4486bc54f4 100644
--- a/apps/web/app/(ee)/api/cron/partner-platforms/website/route.ts
+++ b/apps/web/app/(ee)/api/cron/partner-platforms/website/route.ts
@@ -1,31 +1,21 @@
-import { qstash } from "@/lib/cron";
import { withCron } from "@/lib/cron/with-cron";
import { prisma } from "@dub/prisma";
import { PlatformType } from "@dub/prisma/client";
-import { APP_DOMAIN_WITH_NGROK, chunk, getDomainWithoutWWW } from "@dub/utils";
-import * as z from "zod/v4";
+import { chunk, getDomainWithoutWWW } from "@dub/utils";
import { logAndRespond } from "../../utils";
import { getDomainRating } from "./get-domain-rating";
export const dynamic = "force-dynamic";
-const BATCH_SIZE = 500;
+const BATCH_SIZE = 60;
const CONCURRENCY = 10;
-const schema = z.object({
- startingAfter: z.string().optional(),
-});
-
/**
* This route is used to update domain rating (DR) for verified website partners using the Ahrefs free API
- * Runs once a day at 05:00 AM UTC (cron expression: 0 5 * * *)
- * POST /api/cron/partner-platforms/website
+ * Runs once a minute (cron expression: * * * * *), processing up to 60 platforms per run (Ahrefs rate limit)
+ * GET /api/cron/partner-platforms/website
*/
-export const POST = withCron(async ({ rawBody }) => {
- let { startingAfter } = schema.parse(
- rawBody ? JSON.parse(rawBody) : { startingAfter: undefined },
- );
-
+export const GET = withCron(async () => {
const websites = await prisma.partnerPlatform.findMany({
where: {
type: PlatformType.website,
@@ -34,21 +24,13 @@ export const POST = withCron(async ({ rawBody }) => {
},
},
take: BATCH_SIZE,
- ...(startingAfter && {
- cursor: {
- id: startingAfter,
- },
- skip: 1,
- }),
orderBy: {
- id: "asc",
+ lastCheckedAt: "asc",
},
});
if (websites.length === 0) {
- return logAndRespond(
- "No more website platforms found. Finished updating website domain ratings.",
- );
+ return logAndRespond("No verified website platforms found.");
}
const websiteChunks = chunk(websites, CONCURRENCY);
@@ -69,13 +51,6 @@ export const POST = withCron(async ({ rawBody }) => {
try {
const domainRating = await getDomainRating(target);
- if (website.subscribers === BigInt(domainRating)) {
- console.log(
- `No changes to update for ${target} (DR: ${domainRating}), skipping...`,
- );
- return;
- }
-
await prisma.partnerPlatform.update({
where: {
id: website.id,
@@ -88,6 +63,8 @@ export const POST = withCron(async ({ rawBody }) => {
console.log(`Updated domain rating for ${target}`, {
domainRating,
+ previousDomainRating: Number(website.subscribers),
+ domainRatingChanged: Number(website.subscribers) !== domainRating,
});
} catch (error) {
console.error(`Error updating domain rating for ${target}:`, error);
@@ -96,23 +73,7 @@ export const POST = withCron(async ({ rawBody }) => {
);
}
- if (websites.length === BATCH_SIZE) {
- startingAfter = websites[websites.length - 1].id;
-
- await qstash.publishJSON({
- url: `${APP_DOMAIN_WITH_NGROK}/api/cron/partner-platforms/website`,
- method: "POST",
- body: {
- startingAfter,
- },
- });
-
- return logAndRespond(
- `Processed ${BATCH_SIZE} websites. Scheduled next batch (startingAfter: ${startingAfter}).`,
- );
- }
-
return logAndRespond(
- `Finished updating domain ratings for ${websites.length} websites.`,
+ `Processed domain ratings for ${websites.length} websites.`,
);
});
diff --git a/apps/web/vercel.json b/apps/web/vercel.json
index b9017d7064..9efda2aae5 100644
--- a/apps/web/vercel.json
+++ b/apps/web/vercel.json
@@ -28,6 +28,10 @@
"path": "/api/cron/streams/update-partner-stats",
"schedule": "*/2 * * * *"
},
+ {
+ "path": "/api/cron/partner-platforms/website",
+ "schedule": "* * * * *"
+ },
{
"path": "/api/cron/partner-program-summary",
"schedule": "0 13 1 * *"
diff --git a/packages/prisma/schema/platform.prisma b/packages/prisma/schema/platform.prisma
index 8ffde40a82..f3d5a285bd 100644
--- a/packages/prisma/schema/platform.prisma
+++ b/packages/prisma/schema/platform.prisma
@@ -26,5 +26,6 @@ model PartnerPlatform {
partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade)
@@unique([partnerId, type])
- @@index([type, identifier])
+ @@index([type, verifiedAt, lastCheckedAt])
+ @@index(identifier)
}