Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker/billing): fix crawl overbilling

Gergő Móriczce2f6ff88443
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 1d10fb172..c67512180 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -168,26 +168,6 @@ export async function runWebScraper({
}
if (error === undefined && response?.success) {
- if (is_scrape === false) {
- let creditsToBeBilled = 1; // Assuming 1 credit per document
- if (scrapeOptions.extract) {
- creditsToBeBilled = 5;
- }
-
- // If the team is the background index team, return the response
- if (team_id === process.env.BACKGROUND_INDEX_TEAM_ID!) {
- return response;
- }
-
- billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => {
- logger.error(
- `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,
- { error },
- );
- // Optionally, you could notify an admin or add to a retry queue here
- });
- }
-
return response;
} else {
if (response !== undefined) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 25ab154b8..81ed955ae 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -59,6 +59,7 @@ import { performExtraction } from "../lib/extract/extraction-service";
import { supabase_service } from "../services/supabase";
import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";
import { saveExtract, updateExtract } from "../lib/extract/extract-redis";
+import { billTeam } from "./billing/credit_billing";
configDotenv();
@@ -754,7 +755,6 @@ async function processJob(job: Job & { id: string }, token: string) {
]);
if (!pipeline.success) {
- // TODO: let's Not do this
throw pipeline.error;
}
@@ -961,6 +961,23 @@ async function processJob(job: Job & { id: string }, token: string) {
indexJob(job, doc);
}
+ if (job.data.is_scrape !== true) {
+ let creditsToBeBilled = 1; // Assuming 1 credit per document
+ if (job.data.scrapeOptions.extract) {
+ creditsToBeBilled = 5;
+ }
+
+ if (job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID!) {
+ billTeam(job.data.team_id, undefined, creditsToBeBilled, logger).catch((error) => {
+ logger.error(
+ `Failed to bill team ${job.data.team_id} for ${creditsToBeBilled} credits`,
+ { error },
+ );
+ // Optionally, you could notify an admin or add to a retry queue here
+ });
+ }
+ }
+
logger.info(`🐂 Job done ${job.id}`);
return data;
} catch (error) {