Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/scrape): billing for sites unblocked via flag (#3237)
Gergő Móricz16695709368e
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 9ada777d2..e00adaddf 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -9,9 +9,11 @@ import { CostTracking } from "./cost-tracking";import { hasFormatOfType } from "./format-utils";import { TransportableError } from "./error";import { FeatureFlag } from "../scraper/scrapeURL/engines";+import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;+const unblockedDomainCostBonus = 4;export async function calculateCreditsToBeBilled(options: ScrapeOptions,@@ -94,5 +96,13 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += stealthProxyCostBonus;}+ const urlsToCheck = [+ document.metadata?.url,+ document.metadata?.sourceURL,+ ].filter((u): u is string => !!u);+ if (urlsToCheck.some(u => isUrlBlocked(u, null) && !isUrlBlocked(u, flags))) {+ creditsToBeBilled += unblockedDomainCostBonus;+ }+return creditsToBeBilled;}