Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/scrape): billing for sites unblocked via flag (#3237)

Gergő Móricz16695709368e
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 9ada777d2..e00adaddf 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -9,9 +9,11 @@ import { CostTracking } from "./cost-tracking";
import { hasFormatOfType } from "./format-utils";
import { TransportableError } from "./error";
import { FeatureFlag } from "../scraper/scrapeURL/engines";
+import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
const creditsPerPDFPage = 1;
const stealthProxyCostBonus = 4;
+const unblockedDomainCostBonus = 4;
export async function calculateCreditsToBeBilled(
options: ScrapeOptions,
@@ -94,5 +96,13 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += stealthProxyCostBonus;
}
+ const urlsToCheck = [
+ document.metadata?.url,
+ document.metadata?.sourceURL,
+ ].filter((u): u is string => !!u);
+ if (urlsToCheck.some(u => isUrlBlocked(u, null) && !isUrlBlocked(u, flags))) {
+ creditsToBeBilled += unblockedDomainCostBonus;
+ }
+
return creditsToBeBilled;
}