Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Revert "fix(queue-worker, scrape): match billing logic and add billing for stealth proxies (#1521)""
Gergő Móricz0fd05a67a0c4
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex d052431b6..d492156fc 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -155,6 +155,10 @@ export async function scrapeController(}}+ if (req.body.proxy === "stealth") {+ creditsToBeBilled += 4;+ }+billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch((error) => {logger.error(diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex c6bc71bb1..2a1e47f70 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1355,11 +1355,20 @@ async function processJob(job: Job & { id: string }, token: string) {if (job.data.is_scrape !== true) {let creditsToBeBilled = 1; // Assuming 1 credit per document- if (job.data.scrapeOptions.extract) {+ if ((job.data.scrapeOptions.extract && job.data.scrapeOptions.formats?.includes("extract")) || (job.data.scrapeOptions.formats?.includes("changeTracking") && job.data.scrapeOptions.changeTrackingOptions?.modes?.includes("json"))) {creditsToBeBilled = 5;}- if (job.data.scrapeOptions.agent?.model?.toLowerCase() === "fire-1") {- creditsToBeBilled = 150;++ if (job.data.scrapeOptions.agent?.model?.toLowerCase() === "fire-1" || job.data.scrapeOptions.extract?.agent?.model?.toLowerCase() === "fire-1" || job.data.scrapeOptions.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {+ if (process.env.USE_DB_AUTHENTICATION === "true") {+ creditsToBeBilled = Math.ceil((costTracking.toJSON().totalCost ?? 1) * 1800);+ } else {+ creditsToBeBilled = 150;+ }+ }++ if (job.data.scrapeOptions.proxy === "stealth") {+ creditsToBeBilled += 4;}if (