Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/search): saner billing logic (#2585)
Gergő Móriczb95e4898672d
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 1ee1b3c27..54b5d4ae1 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -20,7 +20,6 @@ import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";import { getJobPriority } from "../../lib/job-priority";import { CostTracking } from "../../lib/cost-tracking";-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";@@ -137,11 +136,10 @@ async function scrapeSearchResult(internalOptions: {...internalOptions,teamId: options.teamId,- bypassBilling: true,+ bypassBilling: false, // Scrape jobs always bill themselveszeroDataRetention,},origin: options.origin,- is_scrape: true,startTime: Date.now(),zeroDataRetention,apiKeyId: options.apiKeyId,@@ -377,59 +375,13 @@ export async function searchController(responseData.data = filteredDocs;}- const finalDocsForBilling = responseData.data;-- const creditPromises = finalDocsForBilling.map(async finalDoc => {- const matchingDocWithCost = docsWithCostTracking.find(- item =>- item.document.metadata &&- finalDoc.metadata &&- item.document.metadata.scrapeId === finalDoc.metadata.scrapeId,- );-- if (matchingDocWithCost) {- const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(- req.body.scrapeOptions,- req.body.timeout,- req.auth.team_id,- );- return await calculateCreditsToBeBilled(- scrapeOptions,- {- ...internalOptions,- teamId: req.auth.team_id,- bypassBilling: true,- zeroDataRetention: false,- },- matchingDocWithCost.document,- matchingDocWithCost.costTracking,- req.acuc?.flags ?? null,- );- } else {- return 1;- }- });-- try {- const individualCredits = await Promise.all(creditPromises);- const scrapeCredits = individualCredits.reduce(- (sum, credit) => sum + credit,- 0,- );-- // Add search credits (same as when no scrape options are specified)- const searchCredits = Math.ceil(responseData.data.length / 10) * 2;-- credits_billed = scrapeCredits + searchCredits;- } catch (error) {- logger.error("Error calculating credits for billing", { error });- credits_billed = responseData.data.length;- }+ // Calculate search credits only - scrape jobs bill themselves+ credits_billed = Math.ceil(responseData.data.length / 10) * 2;allDocsWithCostTracking = docsWithCostTracking;}- // Bill team once for all successful results+ // Bill team for search credits only - scrape jobs handle their own billingif (!isSearchPreview) {billTeam(req.auth.team_id,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex ba2ea4cc1..2b7add643 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -19,7 +19,6 @@ import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";import { getJobPriority } from "../../lib/job-priority";import { CostTracking } from "../../lib/cost-tracking";-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { SearchV2Response } from "../../lib/entities";import { ScrapeJobTimeoutError } from "../../lib/error";@@ -405,7 +404,7 @@ export async function searchController(origin: req.body.origin,timeout: req.body.timeout,scrapeOptions: bodyScrapeOptions,- bypassBilling: !isAsyncScraping || !shouldBill, // Async mode bills per job, sync mode bills manually+ bypassBilling: !shouldBill, // Scrape jobs always bill themselvesapiKeyId: req.acuc?.api_key_id ?? null,zeroDataRetention: isZDROrAnon,requestId: agentRequestId ?? jobId,@@ -657,50 +656,19 @@ export async function searchController(});}- // Calculate credits- // bodyScrapeOptions is guaranteed to exist here because we're in the shouldScrape block- const creditPromises = allDocsWithCostTracking.map(- async docWithCost => {- return await calculateCreditsToBeBilled(- bodyScrapeOptions,- {- teamId: req.auth.team_id,- bypassBilling: true,- zeroDataRetention: isZDROrAnon,- },- docWithCost.document,- docWithCost.costTracking,- req.acuc?.flags ?? null,- );- },- );-- try {- const individualCredits = await Promise.all(creditPromises);- const scrapeCredits = individualCredits.reduce(- (sum, credit) => sum + credit,- 0,- );-- const creditsPerTenResults = isZDR ? 10 : 2;- const searchCredits =- Math.ceil(totalResultsCount / 10) * creditsPerTenResults;-- credits_billed = scrapeCredits + searchCredits;- } catch (error) {- logger.error("Error calculating credits for billing", { error });- credits_billed = totalResultsCount;- }+ // Calculate search credits only - scrape jobs bill themselves+ const creditsPerTenResults = isZDR ? 10 : 2;+ credits_billed =+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults;// Update response with scraped dataObject.assign(searchResponse, scrapedResponse);}}- // Bill team once for all successful results- // - For sync scraping: Bill based on actual scraped content- // - For async scraping: Jobs handle their own billing- // - For no scraping: Bill based on search results count+ // Bill team for search credits only+ // - Scrape jobs always handle their own billing (both sync and async)+ // - Search job only bills for search costs (credits per 10 results)if (!isSearchPreview &&(!shouldScrape || (shouldScrape && !isAsyncScraping))diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 02a5325ff..b97c41dab 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -427,7 +427,9 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) {search.team_id === "preview" || search.team_id?.startsWith("preview_")? previewTeamId: search.team_id,- options: search.zeroDataRetention ? null : search.options,+ options: search.zeroDataRetention+ ? { enterprise: search.options?.enterprise }+ : search.options,credits_cost: search.credits_cost,is_successful: search.is_successful,error: search.zeroDataRetention ? null : (search.error ?? null),