Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker/billing): fix crawl overbilling
Gergő Móriczce2f6ff88443
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 1d10fb172..c67512180 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -168,26 +168,6 @@ export async function runWebScraper({}if (error === undefined && response?.success) {- if (is_scrape === false) {- let creditsToBeBilled = 1; // Assuming 1 credit per document- if (scrapeOptions.extract) {- creditsToBeBilled = 5;- }-- // If the team is the background index team, return the response- if (team_id === process.env.BACKGROUND_INDEX_TEAM_ID!) {- return response;- }-- billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => {- logger.error(- `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,- { error },- );- // Optionally, you could notify an admin or add to a retry queue here- });- }-return response;} else {if (response !== undefined) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 25ab154b8..81ed955ae 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -59,6 +59,7 @@ import { performExtraction } from "../lib/extract/extraction-service";import { supabase_service } from "../services/supabase";import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";import { saveExtract, updateExtract } from "../lib/extract/extract-redis";+import { billTeam } from "./billing/credit_billing";configDotenv();@@ -754,7 +755,6 @@ async function processJob(job: Job & { id: string }, token: string) {]);if (!pipeline.success) {- // TODO: let's Not do thisthrow pipeline.error;}@@ -961,6 +961,23 @@ async function processJob(job: Job & { id: string }, token: string) {indexJob(job, doc);}+ if (job.data.is_scrape !== true) {+ let creditsToBeBilled = 1; // Assuming 1 credit per document+ if (job.data.scrapeOptions.extract) {+ creditsToBeBilled = 5;+ }++ if (job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID!) {+ billTeam(job.data.team_id, undefined, creditsToBeBilled, logger).catch((error) => {+ logger.error(+ `Failed to bill team ${job.data.team_id} for ${creditsToBeBilled} credits`,+ { error },+ );+ // Optionally, you could notify an admin or add to a retry queue here+ });+ }+ }+logger.info(`🐂 Job done ${job.id}`);return data;} catch (error) {