Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(search): scrape index-category results when formats are requested (#5181)
Kris Oei12c2d42875b7
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex a06550cc9..59cc186a9 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -373,16 +373,18 @@ async function searchControllerInner(}const toolsOnly = isToolsOnlySearch(req.body.sources, req.body.categories);+ const govSearch = wantsGovCategory(categories);const projectedKeylessCredits =!isSearchPreview &&shouldBill &&!toolsOnly &&- !wantsGovCategory(categories)+ (!govSearch || requestedFormats.length > 0)? projectSearchTotalCredits({limit: req.body.limit,enterprise: req.body.enterprise,scrapeOptions: req.body.scrapeOptions,+ freeSearch: govSearch,},req.acuc?.flags ?? null,zeroDataRetention,@@ -416,6 +418,10 @@ async function searchControllerInner(if (wantsGovCategory(categories)) {const indexBudget = await reserveGovIndexCalls(req.auth.team_id, 1);if (!indexBudget.ok) {+ if (reservedKeylessCredits > 0) {+ await adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits);+ reconciledKeylessCredits = true;+ }res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));return res.status(429).json({success: false,diff --git a/apps/api/src/lib/keyless-credit-projection.ts b/apps/api/src/lib/keyless-credit-projection.tsindex 0239c4524..e88e9fd1e 100644--- a/apps/api/src/lib/keyless-credit-projection.ts+++ b/apps/api/src/lib/keyless-credit-projection.ts@@ -74,11 +74,14 @@ export function projectSearchTotalCredits(limit: number;enterprise?: ("default" | "anon" | "zdr")[];scrapeOptions?: ScrapeOptions;+ freeSearch?: boolean;},flags: TeamFlags,zeroDataRetention: boolean,): number {- const searchCredits = projectSearchCredits(params.limit, params.enterprise);+ const searchCredits = params.freeSearch+ ? 0+ : projectSearchCredits(params.limit, params.enterprise);const shouldScrape =params.scrapeOptions?.formats && params.scrapeOptions.formats.length > 0;if (!shouldScrape || !params.scrapeOptions) return searchCredits;diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex b0887fa98..77230f8e7 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -207,7 +207,16 @@ export async function executeSearch(!zeroDataRetention &&!options.enterprise?.some(mode => mode === "zdr" || mode === "anon")) {- await removeExplicitResults(searchResponse, limit, logger, teamId);+ // Index results skip the SERP and its provider-side safe search, so they+ // are filtered from a response of their own before they are counted,+ // scraped or returned.+ const safeTarget: SearchV2Response = indexCategorySearch+ ? { web: indexResults }+ : searchResponse;+ await removeExplicitResults(safeTarget, limit, logger, teamId);+ if (safeTarget !== searchResponse) {+ indexResults = safeTarget.web ?? [];+ }}if (searchResponse.web && searchResponse.web.length > 0) {@@ -301,7 +310,12 @@ export async function executeSearch(scrapeOptions?.formats && scrapeOptions.formats.length > 0;if (shouldScrape && scrapeOptions) {- const itemsToScrape = getItemsToScrape(searchResponse, flags, {+ // Index results only become the web group at the end, so they are+ // scraped from a response of their own.+ const scrapeTarget: SearchV2Response = indexCategorySearch+ ? { web: indexResults }+ : searchResponse;+ const itemsToScrape = getItemsToScrape(scrapeTarget, flags, {team_id: teamId,org_id: orgId ?? null,origin,@@ -332,11 +346,10 @@ export async function executeSearch(flags,);- mergeScrapedContent(- searchResponse,- itemsToScrape,- allDocsWithCostTracking,- );+ mergeScrapedContent(scrapeTarget, itemsToScrape, allDocsWithCostTracking);+ if (scrapeTarget !== searchResponse) {+ indexResults = scrapeTarget.web ?? [];+ }scrapeCredits = calculateScrapeCredits(allDocsWithCostTracking);}}