Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import type { Logger } from "winston";
import {
discoverTools,
isAlexandriaSource,
usOnlyToolsWarning,
} from "./alexandria";
import {
discoverSkills,
isSkillsSource,
SKILLS_DISCOVERY_TIMEOUT_MS,
type SkillDiscovery,
} from "./skill-cards";
import {
AUTO_TOOLS_WAIT_MS,
FUSED_FIT_THRESHOLD,
inferToolIntent,
verifyTools,
} from "./tool-intent";
import type { DiscoveredTool } from "../services/alexandria/contracts";
import { search } from "./v2";
import {
SearchV2Response,
SearchResultType,
SearchResultCountsBySource,
SearchResultCategoriesBySource,
countSearchResultsBySource,
collectSearchResultCategories,
} from "../lib/entities";
import {
buildSearchQuery,
getCategoryFromUrl,
CategoryOption,
} from "../lib/search-query-builder";
import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";
import {
getItemsToScrape,
scrapeSearchResults,
mergeScrapedContent,
calculateScrapeCredits,
} from "./scrape";
import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";
import {
MAX_ROUTED_LIMIT,
searchGovCategory,
searchGovIfRouted,
wantsGovCategory,
} from "./gov";
import { removeExplicitResults } from "./safe-search";
import {
matchesQueryOperators,
planOperatorQuery,
siteHostnames,
usesSearchSyntax,
} from "./operators";
import {
highlightsEnvReady,
runIndexedSearchHighlights,
searchHighlightsMode,
} from "./highlights";
import { trackSearchResults, trackSearchRequest } from "../lib/tracking";
import type { BillingMetadata } from "../services/billing/types";
import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";
import { checkUrlsAgainstThreatPolicy } from "../lib/threat-protection/request";
import { calculateThreatScanCredits } from "../lib/scrape-billing";
import { config } from "../config";
import { logger as rootLogger } from "../lib/logger";
import { isUsOnly } from "../lib/us-only";
import { getScrapeZDR } from "../lib/zdr-helpers";
interface SearchOptions {
query: string;
limit: number;
tbs?: string;
filter?: string;
lang?: string;
country?: string;
location?: string;
safe?: boolean;
sources: Array<{ type: string }>;
categories?: CategoryOption[];
includeDomains?: string[];
excludeDomains?: string[];
enterprise?: ("default" | "anon" | "zdr")[];
scrapeOptions?: ScrapeOptions;
highlights?: boolean;
domainTools?: boolean;
toolDetail?: "compact" | "summary" | "full";
/** Rewrite the query before tool discovery. */
inferToolQuery?: boolean;
/** Add compact tools to a search that did not ask for them, when the query needs them. */
autoTools?: boolean;
/** Rank rewrite-found tools together with raw-query tools instead of after them. */
toolFusion?: boolean;
timeout: number;
}
interface SearchContext {
teamId: string;
orgId?: string | null;
origin: string;
integration?: string | null;
apiKeyId: number | null;
flags: TeamFlags;
requestId: string;
jobId: string;
apiVersion: string;
bypassBilling?: boolean;
zeroDataRetention?: boolean;
billing?: BillingMetadata;
agentIndexOnly?: boolean;
keylessReserved?: boolean;
/** Effective threat protection policy; blocked domains are removed from results entirely. */
threatProtectionPolicy?: ThreatProtectionPolicy | null;
/** Set when the request legitimately opted out of Safe Mode at the controller. */
safeModeBypassed?: boolean;
}
interface SearchExecuteResult {
/** Parts of the search a US-only team could not be served. */
usOnlyWarning?: string;
toolsWarning?: string;
/** The tools were added automatically, not requested. */
toolsAuto?: boolean;
toolsUnavailable?: boolean;
/** Raw-query and rewrite tools were ranked together. */
toolFusion: boolean;
/** Skill-card lookup returned the Exchange envelope, including an empty one. */
skillsDiscovered?: boolean;
response: SearchV2Response;
totalResultsCount: number;
resultCountsBySource: SearchResultCountsBySource;
resultCategories: SearchResultCategoriesBySource;
indexResultsCount: number;
searchCredits: number;
scrapeCredits: number;
totalCredits: number;
shouldScrape: boolean;
}
const toolKey = (tool: DiscoveredTool) => `${tool.provider}/${tool.capability}`;
function addTool(tools: Map<string, DiscoveredTool>, tool: DiscoveredTool) {
const key = toolKey(tool);
const previous = tools.get(key);
tools.set(
key,
previous && "matchedBy" in previous && "matchedBy" in tool
? {
...previous,
matchedBy: [
...new Set([
...(previous.matchedBy ?? []),
...(tool.matchedBy ?? []),
]),
],
matchedUrls: [
...new Set([
...(previous.matchedUrls ?? []),
...(tool.matchedUrls ?? []),
]),
],
}
: (previous ?? tool),
);
}
const RRF_K = 60;
/**
* Orders tools by reciprocal rank fusion; ties keep the earlier list's order.
* Tools `keep` rejects are left out without moving the ranks of the rest.
*/
function fuseRankings(
lists: DiscoveredTool[][],
keep: (tool: DiscoveredTool) => boolean,
): DiscoveredTool[] {
const tools = new Map<string, DiscoveredTool>();
const scores = new Map<string, number>();
for (const list of lists)
list.forEach((tool, index) => {
if (!keep(tool)) return;
addTool(tools, tool);
const key = toolKey(tool);
scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K + index + 1));
});
return [...tools.entries()]
.sort(([a], [b]) => scores.get(b)! - scores.get(a)!)
.map(([, tool]) => tool);
}
export async function executeSearch(
options: SearchOptions,
context: SearchContext,
logger: Logger,
): Promise<SearchExecuteResult> {
const { query, limit, sources, categories, scrapeOptions } = options;
const {
teamId,
orgId,
origin,
apiKeyId,
flags,
requestId,
bypassBilling,
zeroDataRetention,
billing,
} = context;
const num_results_buffer = Math.floor(limit * 2);
const usOnly = isUsOnly(flags);
// A team held to zero data retention for scrapes still gets tools, but only from providers
// that support it. Read from team flags only, never from the request.
const toolsZeroDataRetention = getScrapeZDR(flags) === "forced";
const usOnlyWarnings: string[] = [];
logger.info("Searching for results");
const wantsTools = sources.some(isAlexandriaSource);
const wantsSkills = sources.some(isSkillsSource);
const toolsAllowed =
!zeroDataRetention &&
!options.enterprise?.some(mode => mode === "zdr" || mode === "anon");
const missedSkills: SkillDiscovery = { skills: [], succeeded: false };
const skillsPromise =
toolsAllowed && wantsSkills && query.trim()
? discoverSkills(
{
teamId,
query,
timeoutMs: Math.min(options.timeout, SKILLS_DISCOVERY_TIMEOUT_MS),
usOnly,
zeroDataRetention: toolsZeroDataRetention,
},
logger,
).catch(error => {
logger.warn("Skill discovery failed", { error });
return missedSkills;
})
: null;
const toolIntent =
toolsAllowed &&
(wantsTools ? options.inferToolQuery : options.autoTools) &&
query.trim()
? inferToolIntent(query, logger)
: null;
const fuseTools = wantsTools && options.toolFusion === true;
const toolRequestIds = {
raw: { prefix: `${context.jobId}:tools:raw`, sent: [] as string[] },
rewrite: { prefix: `${context.jobId}:tools:rewrite`, sent: [] as string[] },
};
// Tools found only through the rewrite or inferred domains are added after Jev confirms they fit the query.
const extraDiscovery = toolIntent
? toolIntent
.then(async intent => {
if (!wantsTools && !intent.needsTools) return null;
const found = await discoverTools(
{
toolDetail: options.toolDetail ?? "compact",
teamId,
connectors: flags?.exchangeRetrieve === true,
usOnly,
zeroDataRetention: toolsZeroDataRetention,
limit,
queries: wantsTools
? intent.toolQueries
: [query, ...intent.toolQueries],
urls:
options.domainTools === false
? []
: intent.domains.map(domain => `https://${domain}/`),
timeoutMs: Math.min(options.timeout, 3000),
exchangeRequestIds: toolRequestIds.rewrite,
},
logger,
);
return {
// Only fused semantic matches get the lower bar; domain matches keep the default.
items: fuseTools
? (
await Promise.all([
verifyTools(
query,
found.items.slice(0, found.semanticMatches),
logger,
FUSED_FIT_THRESHOLD,
),
found.items.length > found.semanticMatches
? verifyTools(
query,
found.items.slice(found.semanticMatches),
logger,
)
: [],
])
).flat()
: await verifyTools(query, found.items, logger),
found: found.items,
semanticMatches: found.semanticMatches,
usOnlyExcluded: found.usOnlyExcluded ?? 0,
};
})
.catch(() => null)
: null;
let resolveToolUrls: (urls: string[]) => void = () => {};
const baseDiscovery =
toolsAllowed && (wantsTools || options.domainTools)
? discoverTools(
{
toolDetail: options.toolDetail ?? "compact",
teamId,
connectors: flags?.exchangeRetrieve === true,
usOnly,
zeroDataRetention: toolsZeroDataRetention,
limit,
queries: wantsTools ? [query] : undefined,
urls:
options.domainTools === false
? []
: new Promise<string[]>(resolve => {
resolveToolUrls = resolve;
}),
timeoutMs: options.timeout,
exchangeRequestIds: toolRequestIds.raw,
},
logger,
)
: null;
const searchTypes = [
...new Set(
sources
.filter(
source => !isAlexandriaSource(source) && !isSkillsSource(source),
)
.map(source => source.type as SearchResultType),
),
];
const operatorPlan = planOperatorQuery(query);
const { operators } = operatorPlan;
const nonEmpty = (list: string[]) => (list.length > 0 ? list : undefined);
const { query: searchQuery, categoryMap } = buildSearchQuery(
operatorPlan.query,
categories,
{
includeDomains: options.includeDomains,
excludeDomains: options.excludeDomains,
},
);
// The developer and gov categories are each exclusive (schema-enforced) and
// are served from their own index instead of the web SERP.
const indexCategorySearch = wantsDeveloperCategory(categories)
? searchDeveloperCategory
: wantsGovCategory(categories)
? searchGovCategory
: null;
const indexSearchOptions = {
query,
limit,
teamId,
timeout: options.timeout,
zeroDataRetention:
zeroDataRetention === true ||
options.enterprise?.some(mode => mode === "zdr" || mode === "anon") ===
true,
...(usOnly ? { usOnly, warnings: usOnlyWarnings } : {}),
};
// A plain US web search that the search platform classifies as government
// is served through the Government Index. Zero data retention and
// anonymous searches are never routed, as they skip every other auxiliary
// service here, and neither is a page larger than MAX_ROUTED_LIMIT.
const plainWebSearch =
indexCategorySearch === null &&
!indexSearchOptions.zeroDataRetention &&
!categories?.length &&
searchTypes.length === 1 &&
searchTypes[0] === "web" &&
!options.includeDomains?.length &&
!options.excludeDomains?.length &&
!options.tbs &&
!options.filter &&
(options.country ?? "us").toLowerCase() === "us" &&
!options.location &&
(options.lang ?? "en").toLowerCase() === "en" &&
!usOnly &&
query.trim().length > 0 &&
limit <= MAX_ROUTED_LIMIT &&
!usesSearchSyntax(query);
// A plain web search takes the routing decision before the regular web
// search, so a routed search never starts it. An unrouted one, or a routed
// one whose platform search fails, starts the regular web search after the
// decision.
const routedResults = plainWebSearch
? await searchGovIfRouted(indexSearchOptions, context.jobId, logger)
: null;
const webSearchPromise =
routedResults !== null ||
indexCategorySearch !== null ||
((wantsTools || wantsSkills) && !searchTypes.length)
? null
: search({
query: searchQuery,
logger,
requestId: context.requestId,
advanced: false,
num_results: operatorPlan.relaxed
? Math.max(num_results_buffer, Math.min(limit * 5, 100))
: num_results_buffer,
tbs: options.tbs,
filter: options.filter,
lang: options.lang,
country: options.country,
location: options.location,
safe: options.safe,
type: searchTypes,
enterprise: options.enterprise,
includeDomains: options.includeDomains?.length
? options.includeDomains
: nonEmpty(siteHostnames(operators.sites)),
excludeDomains: nonEmpty([
...(options.excludeDomains ?? []),
...operators.excludedSites.filter(site => !site.includes("/")),
]),
usOnly,
});
const indexResultsPromise = indexCategorySearch
? indexCategorySearch(indexSearchOptions, logger)
: null;
// Routed results take the regular web search's place as the web results, so
// threat protection, safe search, categories, highlights, scraping and
// billing treat them as a web search's.
const searchResponse: SearchV2Response = routedResults
? { web: routedResults }
: webSearchPromise === null || indexResultsPromise !== null
? ({} as SearchV2Response)
: ((await webSearchPromise) as SearchV2Response);
const keepOperatorMatches = <
T extends { url?: string; title?: string; position?: number },
>(
results: T[] | undefined,
): T[] | undefined => {
const kept = results?.filter(result =>
matchesQueryOperators(result, operatorPlan),
);
return kept?.length === results?.length
? results
: kept?.map((result, index) => ({ ...result, position: index + 1 }));
};
searchResponse.web = keepOperatorMatches(searchResponse.web);
searchResponse.news = keepOperatorMatches(searchResponse.news);
searchResponse.images = keepOperatorMatches(searchResponse.images);
let indexResults =
keepOperatorMatches(
indexResultsPromise ? await indexResultsPromise : undefined,
) ?? [];
// Threat protection: remove blocked results entirely — before
// slicing/counting, before scraping, and before returning. Checks are
// URL-level and deduped within this request; scan fees bill +2 per unique
// scanned URL (see calculateThreatScanCredits), charged as part of the
// search credits below.
let threatScanCredits = 0;
const threatPolicy = context.threatProtectionPolicy;
if (threatPolicy && threatPolicy.mode !== "off") {
const urlsToCheck = [
...(searchResponse.web ?? []).map(x => x.url),
...(searchResponse.news ?? []).map(x => x.url),
...(searchResponse.images ?? []).map(x => x.url),
...indexResults.map(x => x.url),
].filter((x): x is string => !!x);
if (urlsToCheck.length > 0) {
const { decisionsByUrl } = await checkUrlsAgainstThreatPolicy(
urlsToCheck,
threatPolicy,
{ teamId },
);
threatScanCredits = calculateThreatScanCredits(decisionsByUrl.values());
const isAllowed = (url: string | undefined | null): boolean => {
if (!url) return true;
const decision = decisionsByUrl.get(url);
return decision === undefined || decision.allowed;
};
if (searchResponse.web) {
searchResponse.web = searchResponse.web.filter(x => isAllowed(x.url));
}
if (searchResponse.news) {
searchResponse.news = searchResponse.news.filter(x => isAllowed(x.url));
}
if (searchResponse.images) {
searchResponse.images = searchResponse.images.filter(x =>
isAllowed(x.url),
);
}
indexResults = indexResults.filter(x => isAllowed(x.url));
}
}
// The filter shares results with TypeSafe, so zero data retention and
// anonymous requests skip it.
if (
options.safe &&
!zeroDataRetention &&
!options.enterprise?.some(mode => mode === "zdr" || mode === "anon")
) {
// Index results skip the SERP and its provider-side safe search, so they
// are filtered from a response of their own before they are counted,
// scraped or returned.
const safeTarget: SearchV2Response = indexCategorySearch
? { web: indexResults }
: searchResponse;
await removeExplicitResults(safeTarget, limit, logger, teamId);
if (safeTarget !== searchResponse) {
indexResults = safeTarget.web ?? [];
}
}
if (searchResponse.web && searchResponse.web.length > 0) {
searchResponse.web = searchResponse.web.map(result => ({
...result,
category: getCategoryFromUrl(result.url, categoryMap),
}));
}
if (searchResponse.news && searchResponse.news.length > 0) {
searchResponse.news = searchResponse.news.map(result => ({
...result,
category: result.url
? getCategoryFromUrl(result.url, categoryMap)
: undefined,
}));
}
let totalResultsCount = 0;
if (searchResponse.web && searchResponse.web.length > 0) {
if (searchResponse.web.length > limit) {
searchResponse.web = searchResponse.web.slice(0, limit);
}
totalResultsCount += searchResponse.web.length;
}
if (searchResponse.images && searchResponse.images.length > 0) {
if (searchResponse.images.length > limit) {
searchResponse.images = searchResponse.images.slice(0, limit);
}
totalResultsCount += searchResponse.images.length;
}
if (searchResponse.news && searchResponse.news.length > 0) {
if (searchResponse.news.length > limit) {
searchResponse.news = searchResponse.news.slice(0, limit);
}
totalResultsCount += searchResponse.news.length;
}
const indexResultsCount = indexResults.length;
totalResultsCount += indexResultsCount;
let toolsWarning: string | undefined;
let toolsUnavailable = false;
let toolsAuto = false;
let rewriteLate = false;
let rawTools: Awaited<typeof baseDiscovery> = null;
let extra: Awaited<typeof extraDiscovery> = null;
const waitForExtra = async (alongside?: Promise<unknown>) => {
if (!extraDiscovery) return null;
let timer: NodeJS.Timeout | undefined;
let settled = false;
const delay = (ms: number) =>
new Promise<void>(resolve => {
if (settled) return resolve();
timer = setTimeout(resolve, ms);
timer.unref();
});
// Fused tools are worth waiting for past the raw lookup; appended ones are not.
const deadline = fuseTools
? Promise.resolve(alongside).then(() =>
delay(config.ALEXANDRIA_TOOL_FUSION_WAIT_MS),
)
: Promise.all([delay(AUTO_TOOLS_WAIT_MS), alongside]);
const late = Symbol("late");
try {
const found = await Promise.race([
extraDiscovery,
deadline.then((): typeof late => late),
]);
if (found !== late) return found;
rewriteLate = true;
return null;
} finally {
settled = true;
clearTimeout(timer);
}
};
if (baseDiscovery) {
resolveToolUrls(
[
...(searchResponse.web ?? []),
...(searchResponse.news ?? []),
...indexResults,
].flatMap(item => (item.url ? [item.url] : [])),
);
const [discovery, fetched] = await Promise.all([
baseDiscovery,
waitForExtra(baseDiscovery),
]);
rawTools = discovery;
extra = fetched;
const merged = new Map<string, DiscoveredTool>();
if (fuseTools && extra) {
// Semantic matches are fused; domain-only matches follow, as they do unfused.
const fits = new Set(extra.items.map(toolKey));
const baseSemantic = discovery.items.slice(0, discovery.semanticMatches);
const baseKeys = new Set(baseSemantic.map(toolKey));
for (const tool of [
...fuseRankings(
[baseSemantic, extra.found.slice(0, extra.semanticMatches)],
tool => baseKeys.has(toolKey(tool)) || fits.has(toolKey(tool)),
),
...discovery.items.slice(baseSemantic.length),
...extra.found
.slice(extra.semanticMatches)
.filter(tool => fits.has(toolKey(tool))),
])
addTool(merged, tool);
} else {
for (const tool of [...discovery.items, ...(extra?.items ?? [])])
addTool(merged, tool);
}
searchResponse.tools = [...merged.values()];
// Both legs can hide the same tools, so the larger count is reported.
const usOnlyExcluded = Math.max(
discovery.usOnlyExcluded ?? 0,
extra?.usOnlyExcluded ?? 0,
);
toolsWarning =
[
discovery.warning,
usOnlyExcluded > 0 ? usOnlyToolsWarning(usOnlyExcluded) : undefined,
]
.filter(Boolean)
.join(" ") || undefined;
toolsUnavailable = discovery.unavailable === true && !extra?.items.length;
} else if (extraDiscovery) {
extra = await waitForExtra();
if (extra?.items.length) {
searchResponse.tools = extra.items;
toolsAuto = true;
}
if (extra?.usOnlyExcluded) {
toolsWarning = usOnlyToolsWarning(extra.usOnlyExcluded);
}
}
const rewriteFound = (extra?.found.length ?? 0) > 0;
const toolFusionRan = fuseTools && baseDiscovery !== null && rewriteFound;
if (searchResponse.tools?.length) {
const rawKeys = new Set(rawTools?.items.map(toolKey));
const fitKeys = new Set(extra?.items.map(toolKey));
const rejected = (extra?.found ?? []).filter(
tool => !fitKeys.has(toolKey(tool)) && !rawKeys.has(toolKey(tool)),
);
const semanticKeys = new Set(
extra?.found.slice(0, extra.semanticMatches).map(toolKey),
);
// Root logger: the request logger carries the query, and this line records shape only.
rootLogger.info("Search tool impression", {
event: "search_tool_impression",
searchId: context.jobId,
teamId,
servedAt: new Date().toISOString(),
auto: toolsAuto,
fusion: { enabled: fuseTools, ran: toolFusionRan },
rewrite: !extraDiscovery
? "off"
: rewriteFound
? "arrived"
: rewriteLate
? "late"
: "empty",
tools: searchResponse.tools.map((tool, index) => {
const key = toolKey(tool);
const raw = rawKeys.has(key);
const rewrite = fitKeys.has(key);
return {
id: key,
rank: index + 1,
source: raw && rewrite ? "both" : raw ? "raw" : "rewrite",
};
}),
fitRejected: {
semantic: rejected.filter(tool => semanticKeys.has(toolKey(tool)))
.length,
domain: rejected.filter(tool => !semanticKeys.has(toolKey(tool)))
.length,
},
exchangeRequestIds: {
raw: [...toolRequestIds.raw.sent],
rewrite: [...toolRequestIds.rewrite.sent],
},
});
}
const isZDR = options.enterprise?.includes("zdr");
const creditsPerTenResults = isZDR ? 10 : 2;
// Gov index results are free; developer index results bill like web results.
const billableResultsCount =
indexCategorySearch === searchGovCategory
? totalResultsCount - indexResultsCount
: totalResultsCount;
// Threat protection scan fees ride on the search credits: they are part of
// serving the search itself (every result domain is scanned before
// filtering), so they bill against the same feature and show up in the
// request's creditsUsed.
const searchCredits =
Math.ceil(billableResultsCount / 10) * creditsPerTenResults +
threatScanCredits;
let scrapeCredits = 0;
const shouldScrape =
scrapeOptions?.formats && scrapeOptions.formats.length > 0;
if (shouldScrape && scrapeOptions) {
// Index results only become the web group at the end, so they are
// scraped from a response of their own.
const scrapeTarget: SearchV2Response = indexCategorySearch
? { web: indexResults }
: searchResponse;
const itemsToScrape = getItemsToScrape(scrapeTarget, flags, {
team_id: teamId,
org_id: orgId ?? null,
origin,
});
if (itemsToScrape.length > 0) {
const scrapeOpts = {
teamId,
orgId: orgId ?? null,
origin,
timeout: options.timeout,
scrapeOptions,
bypassBilling: bypassBilling ?? false,
apiKeyId,
zeroDataRetention,
requestId,
billing,
agentIndexOnly: context.agentIndexOnly,
keylessReserved: context.keylessReserved,
threatProtectionPolicy: threatPolicy ?? null,
safeModeBypassed: context.safeModeBypassed ?? false,
};
const allDocsWithCostTracking = await scrapeSearchResults(
itemsToScrape.map(i => i.scrapeInput),
scrapeOpts,
logger,
flags,
);
mergeScrapedContent(scrapeTarget, itemsToScrape, allDocsWithCostTracking);
if (scrapeTarget !== searchResponse) {
indexResults = scrapeTarget.web ?? [];
}
scrapeCredits = calculateScrapeCredits(allDocsWithCostTracking);
}
}
// Every eligible request runs the same index-backed highlight path. MCP/CLI,
// explicit opt-ins, and rollout-selected cohorts await and apply the result;
// everyone else runs it in shadow without delaying or mutating the response.
// Runs after scraping (mergeScrapedContent rebuilds the result objects, so
// highlight mutations must come last to survive). Uses the user's original
// query, not the domain-filtered upstream query.
if (zeroDataRetention !== true && !isZDR && highlightsEnvReady()) {
const mode = searchHighlightsMode({
requested: options.highlights,
origin: context.origin,
integration: context.integration,
cohortKey:
context.apiKeyId !== null
? `api-key:${context.apiKeyId}`
: `team:${context.teamId}`,
rolloutPercent: config.HIGHLIGHT_ROLLOUT_PERCENT,
});
const highlightRun = runIndexedSearchHighlights(
searchResponse,
query,
logger,
{
mode,
requestId: context.requestId,
teamId: context.teamId,
},
);
if (mode === "apply") {
await highlightRun;
} else {
void highlightRun.catch(error => {
rootLogger.warn("Search highlights shadow failed", {
canonicalLog: "search/highlights",
mode,
requestId: context.requestId,
teamId: context.teamId,
errorType: error instanceof Error ? error.name : "unknown",
});
});
}
}
if (indexResultsPromise !== null) {
// An index category is exclusive, so these are the only results: they ARE
// the web group. Threat filtering above may have removed entries, so
// renumber the survivors.
searchResponse.web = indexResults.map((result, index) => ({
...result,
position: index + 1,
}));
}
let skillsDiscovered = false;
if (skillsPromise) {
const found = await skillsPromise;
searchResponse.skills = found.skills;
skillsDiscovered = found.succeeded;
}
const scrapeFormats = scrapeOptions?.formats
? scrapeOptions.formats.map((f: any) =>
typeof f === "string" ? f : f.type,
)
: [];
trackSearchRequest({
searchId: context.jobId,
requestId: context.requestId,
teamId,
query,
origin,
kind: billing?.endpoint ?? "search",
apiVersion: context.apiVersion,
lang: options.lang,
country: options.country,
sources: searchTypes,
numResults: totalResultsCount,
searchCredits,
scrapeCredits,
totalCredits: searchCredits + scrapeCredits,
hasScrapeFormats: shouldScrape ?? false,
scrapeFormats,
isSuccessful: true,
timeTaken: 0, // filled by caller if needed
zeroDataRetention: zeroDataRetention ?? false,
}).catch(err =>
logger.warn("Search request tracking failed", { error: err }),
);
trackSearchResults({
searchId: context.jobId,
teamId,
response: searchResponse,
zeroDataRetention: zeroDataRetention ?? false,
hasScrapeFormats: shouldScrape ?? false,
}).catch(err => logger.warn("Search tracking failed", { error: err }));
return {
response: searchResponse,
...(usOnlyWarnings.length > 0
? { usOnlyWarning: usOnlyWarnings.join(" ") }
: {}),
toolsWarning,
toolsUnavailable,
toolsAuto,
toolFusion: toolFusionRan,
skillsDiscovered,
totalResultsCount,
// Counted from the final response — after scraping and highlights — so it
// matches exactly what the client can address by position.
resultCountsBySource: countSearchResultsBySource(searchResponse),
// Read from the same final response, so a category is keyed by the position
// the client sees. Index hits replace the web group above and are
// renumbered there, so their positions here are the renumbered ones.
resultCategories: collectSearchResultCategories(searchResponse),
indexResultsCount,
searchCredits,
scrapeCredits,
totalCredits: searchCredits + scrapeCredits,
shouldScrape: shouldScrape ?? false,
};
}