Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/search): saner billing logic (#2585)

Gergő Móriczb95e4898672d
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 1ee1b3c27..54b5d4ae1 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -20,7 +20,6 @@ import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { getJobPriority } from "../../lib/job-priority";
import { CostTracking } from "../../lib/cost-tracking";
-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
@@ -137,11 +136,10 @@ async function scrapeSearchResult(
internalOptions: {
...internalOptions,
teamId: options.teamId,
- bypassBilling: true,
+ bypassBilling: false, // Scrape jobs always bill themselves
zeroDataRetention,
},
origin: options.origin,
- is_scrape: true,
startTime: Date.now(),
zeroDataRetention,
apiKeyId: options.apiKeyId,
@@ -377,59 +375,13 @@ export async function searchController(
responseData.data = filteredDocs;
}
- const finalDocsForBilling = responseData.data;
-
- const creditPromises = finalDocsForBilling.map(async finalDoc => {
- const matchingDocWithCost = docsWithCostTracking.find(
- item =>
- item.document.metadata &&
- finalDoc.metadata &&
- item.document.metadata.scrapeId === finalDoc.metadata.scrapeId,
- );
-
- if (matchingDocWithCost) {
- const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(
- req.body.scrapeOptions,
- req.body.timeout,
- req.auth.team_id,
- );
- return await calculateCreditsToBeBilled(
- scrapeOptions,
- {
- ...internalOptions,
- teamId: req.auth.team_id,
- bypassBilling: true,
- zeroDataRetention: false,
- },
- matchingDocWithCost.document,
- matchingDocWithCost.costTracking,
- req.acuc?.flags ?? null,
- );
- } else {
- return 1;
- }
- });
-
- try {
- const individualCredits = await Promise.all(creditPromises);
- const scrapeCredits = individualCredits.reduce(
- (sum, credit) => sum + credit,
- 0,
- );
-
- // Add search credits (same as when no scrape options are specified)
- const searchCredits = Math.ceil(responseData.data.length / 10) * 2;
-
- credits_billed = scrapeCredits + searchCredits;
- } catch (error) {
- logger.error("Error calculating credits for billing", { error });
- credits_billed = responseData.data.length;
- }
+ // Calculate search credits only - scrape jobs bill themselves
+ credits_billed = Math.ceil(responseData.data.length / 10) * 2;
allDocsWithCostTracking = docsWithCostTracking;
}
- // Bill team once for all successful results
+ // Bill team for search credits only - scrape jobs handle their own billing
if (!isSearchPreview) {
billTeam(
req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index ba2ea4cc1..2b7add643 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -19,7 +19,6 @@ import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { getJobPriority } from "../../lib/job-priority";
import { CostTracking } from "../../lib/cost-tracking";
-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { SearchV2Response } from "../../lib/entities";
import { ScrapeJobTimeoutError } from "../../lib/error";
@@ -405,7 +404,7 @@ export async function searchController(
origin: req.body.origin,
timeout: req.body.timeout,
scrapeOptions: bodyScrapeOptions,
- bypassBilling: !isAsyncScraping || !shouldBill, // Async mode bills per job, sync mode bills manually
+ bypassBilling: !shouldBill, // Scrape jobs always bill themselves
apiKeyId: req.acuc?.api_key_id ?? null,
zeroDataRetention: isZDROrAnon,
requestId: agentRequestId ?? jobId,
@@ -657,50 +656,19 @@ export async function searchController(
});
}
- // Calculate credits
- // bodyScrapeOptions is guaranteed to exist here because we're in the shouldScrape block
- const creditPromises = allDocsWithCostTracking.map(
- async docWithCost => {
- return await calculateCreditsToBeBilled(
- bodyScrapeOptions,
- {
- teamId: req.auth.team_id,
- bypassBilling: true,
- zeroDataRetention: isZDROrAnon,
- },
- docWithCost.document,
- docWithCost.costTracking,
- req.acuc?.flags ?? null,
- );
- },
- );
-
- try {
- const individualCredits = await Promise.all(creditPromises);
- const scrapeCredits = individualCredits.reduce(
- (sum, credit) => sum + credit,
- 0,
- );
-
- const creditsPerTenResults = isZDR ? 10 : 2;
- const searchCredits =
- Math.ceil(totalResultsCount / 10) * creditsPerTenResults;
-
- credits_billed = scrapeCredits + searchCredits;
- } catch (error) {
- logger.error("Error calculating credits for billing", { error });
- credits_billed = totalResultsCount;
- }
+ // Calculate search credits only - scrape jobs bill themselves
+ const creditsPerTenResults = isZDR ? 10 : 2;
+ credits_billed =
+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults;
// Update response with scraped data
Object.assign(searchResponse, scrapedResponse);
}
}
- // Bill team once for all successful results
- // - For sync scraping: Bill based on actual scraped content
- // - For async scraping: Jobs handle their own billing
- // - For no scraping: Bill based on search results count
+ // Bill team for search credits only
+ // - Scrape jobs always handle their own billing (both sync and async)
+ // - Search job only bills for search costs (credits per 10 results)
if (
!isSearchPreview &&
(!shouldScrape || (shouldScrape && !isAsyncScraping))
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 02a5325ff..b97c41dab 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -427,7 +427,9 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {
search.team_id === "preview" || search.team_id?.startsWith("preview_")
? previewTeamId
: search.team_id,
- options: search.zeroDataRetention ? null : search.options,
+ options: search.zeroDataRetention
+ ? { enterprise: search.options?.enterprise }
+ : search.options,
credits_cost: search.credits_cost,
is_successful: search.is_successful,
error: search.zeroDataRetention ? null : (search.error ?? null),