Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(search): scrape index-category results when formats are requested (#5181)

Kris Oei12c2d42875b7
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index a06550cc9..59cc186a9 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -373,16 +373,18 @@ async function searchControllerInner(
}
const toolsOnly = isToolsOnlySearch(req.body.sources, req.body.categories);
+ const govSearch = wantsGovCategory(categories);
const projectedKeylessCredits =
!isSearchPreview &&
shouldBill &&
!toolsOnly &&
- !wantsGovCategory(categories)
+ (!govSearch || requestedFormats.length > 0)
? projectSearchTotalCredits(
{
limit: req.body.limit,
enterprise: req.body.enterprise,
scrapeOptions: req.body.scrapeOptions,
+ freeSearch: govSearch,
},
req.acuc?.flags ?? null,
zeroDataRetention,
@@ -416,6 +418,10 @@ async function searchControllerInner(
if (wantsGovCategory(categories)) {
const indexBudget = await reserveGovIndexCalls(req.auth.team_id, 1);
if (!indexBudget.ok) {
+ if (reservedKeylessCredits > 0) {
+ await adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits);
+ reconciledKeylessCredits = true;
+ }
res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));
return res.status(429).json({
success: false,
diff --git a/apps/api/src/lib/keyless-credit-projection.ts b/apps/api/src/lib/keyless-credit-projection.ts
index 0239c4524..e88e9fd1e 100644
--- a/apps/api/src/lib/keyless-credit-projection.ts
+++ b/apps/api/src/lib/keyless-credit-projection.ts
@@ -74,11 +74,14 @@ export function projectSearchTotalCredits(
limit: number;
enterprise?: ("default" | "anon" | "zdr")[];
scrapeOptions?: ScrapeOptions;
+ freeSearch?: boolean;
},
flags: TeamFlags,
zeroDataRetention: boolean,
): number {
- const searchCredits = projectSearchCredits(params.limit, params.enterprise);
+ const searchCredits = params.freeSearch
+ ? 0
+ : projectSearchCredits(params.limit, params.enterprise);
const shouldScrape =
params.scrapeOptions?.formats && params.scrapeOptions.formats.length > 0;
if (!shouldScrape || !params.scrapeOptions) return searchCredits;
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index b0887fa98..77230f8e7 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -207,7 +207,16 @@ export async function executeSearch(
!zeroDataRetention &&
!options.enterprise?.some(mode => mode === "zdr" || mode === "anon")
) {
- await removeExplicitResults(searchResponse, limit, logger, teamId);
+ // Index results skip the SERP and its provider-side safe search, so they
+ // are filtered from a response of their own before they are counted,
+ // scraped or returned.
+ const safeTarget: SearchV2Response = indexCategorySearch
+ ? { web: indexResults }
+ : searchResponse;
+ await removeExplicitResults(safeTarget, limit, logger, teamId);
+ if (safeTarget !== searchResponse) {
+ indexResults = safeTarget.web ?? [];
+ }
}
if (searchResponse.web && searchResponse.web.length > 0) {
@@ -301,7 +310,12 @@ export async function executeSearch(
scrapeOptions?.formats && scrapeOptions.formats.length > 0;
if (shouldScrape && scrapeOptions) {
- const itemsToScrape = getItemsToScrape(searchResponse, flags, {
+ // Index results only become the web group at the end, so they are
+ // scraped from a response of their own.
+ const scrapeTarget: SearchV2Response = indexCategorySearch
+ ? { web: indexResults }
+ : searchResponse;
+ const itemsToScrape = getItemsToScrape(scrapeTarget, flags, {
team_id: teamId,
org_id: orgId ?? null,
origin,
@@ -332,11 +346,10 @@ export async function executeSearch(
flags,
);
- mergeScrapedContent(
- searchResponse,
- itemsToScrape,
- allDocsWithCostTracking,
- );
+ mergeScrapedContent(scrapeTarget, itemsToScrape, allDocsWithCostTracking);
+ if (scrapeTarget !== searchResponse) {
+ indexResults = scrapeTarget.web ?? [];
+ }
scrapeCredits = calculateScrapeCredits(allDocsWithCostTracking);
}
}