Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixed the batch scrape + llm extract billing
Nicolas0bad43606119
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 8eb679e73..8bd0c12c1 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -121,8 +121,13 @@ export async function runWebScraper({: docs;if(is_scrape === false) {- billTeam(team_id, undefined, filteredDocs.length).catch(error => {- Logger.error(`Failed to bill team ${team_id} for ${filteredDocs.length} credits: ${error}`);+ let creditsToBeBilled = 1; // Assuming 1 credit per document+ if (extractorOptions && (extractorOptions.mode === "llm-extraction" || extractorOptions.mode === "extract")) {+ creditsToBeBilled = 5;+ }++ billTeam(team_id, undefined, creditsToBeBilled * filteredDocs.length).catch(error => {+ Logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled * filteredDocs.length} credits: ${error}`);// Optionally, you could notify an admin or add to a retry queue here});}