Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(feat/search) Enterprise params (#2412)
Nicolasaf2db8d046bf
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 1ddf63799..1058fe359 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -391,10 +391,15 @@ export async function searchController(try {const individualCredits = await Promise.all(creditPromises);- credits_billed = individualCredits.reduce(+ const scrapeCredits = individualCredits.reduce((sum, credit) => sum + credit,0,);++ // Add search credits (same as when no scrape options are specified)+ const searchCredits = Math.ceil(responseData.data.length / 10) * 2;++ credits_billed = scrapeCredits + searchCredits;} catch (error) {logger.error("Error calculating credits for billing", { error });credits_billed = responseData.data.length;diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex c24c697fb..4b6534119 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -51,6 +51,7 @@ async function startScrapeJob(scrapeOptions: ScrapeOptions;bypassBilling?: boolean;apiKeyId: number | null;+ zeroDataRetention?: boolean;},logger: Logger,flags: TeamFlags,@@ -59,7 +60,8 @@ async function startScrapeJob(): Promise<string> {const jobId = uuidv7();- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention =+ options.zeroDataRetention ?? flags?.forceZDR ?? false;logger.info("Adding scrape job", {scrapeId: jobId,@@ -114,6 +116,7 @@ async function scrapeSearchResult(scrapeOptions: ScrapeOptions;bypassBilling?: boolean;apiKeyId: number | null;+ zeroDataRetention?: boolean;},logger: Logger,flags: TeamFlags,@@ -250,6 +253,10 @@ export async function searchController(// After transformation, sources is always an array of objectsconst searchTypes = [...new Set(req.body.sources.map((s: any) => s.type))];+ const isZDR = req.body.enterprise?.includes("zdr");+ const isAnon = req.body.enterprise?.includes("anon");+ const isZDROrAnon = isZDR || isAnon;+// Build search query with category filtersconst { query: searchQuery, categoryMap } = buildSearchQuery(req.body.query,@@ -267,6 +274,7 @@ export async function searchController(country: req.body.country,location: req.body.location,type: searchTypes,+ enterprise: req.body.enterprise,})) as SearchV2Response;// Add category labels to web results@@ -321,8 +329,8 @@ export async function searchController(const isAsyncScraping = req.body.asyncScraping && shouldScrape;if (!shouldScrape) {- // No scraping - 2 credits per 10 search results- credits_billed = Math.ceil(totalResultsCount / 10) * 2;+ const creditsPerTenResults = isZDR ? 10 : 2;+ credits_billed = Math.ceil(totalResultsCount / 10) * creditsPerTenResults;} else {// Common setup for both async and sync scrapinglogger.info(@@ -337,6 +345,7 @@ export async function searchController(scrapeOptions: req.body.scrapeOptions,bypassBilling: !isAsyncScraping, // Async mode bills per job, sync mode bills manuallyapiKeyId: req.acuc?.api_key_id ?? null,+ zeroDataRetention: isZDROrAnon,};const directToBullMQ = (req.acuc?.price_credits ?? 0) <= 3000;@@ -472,8 +481,23 @@ export async function searchController();}- // Don't bill here - let each job bill itself when it completes- credits_billed = allJobIds.length; // Just for reporting, not billing+ const creditsPerTenResults = isZDR ? 10 : 2;+ credits_billed =+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults;++ // Bill for search results now (scrape jobs will bill themselves when they complete)+ if (!isSearchPreview) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ credits_billed,+ req.acuc?.api_key_id ?? null,+ ).catch(error => {+ logger.error(+ `Failed to bill team ${req.acuc?.sub_id} for ${credits_billed} credits: ${error}`,+ );+ });+ }const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;@@ -483,34 +507,35 @@ export async function searchController(time_taken: timeTakenInSeconds,scrapeIds,});-- logJob(- {- job_id: jobId,- success: true,- num_docs:- (searchResponse.web?.length ?? 0) +- (searchResponse.images?.length ?? 0) +- (searchResponse.news?.length ?? 0),- docs: [searchResponse],- time_taken: timeTakenInSeconds,- team_id: req.auth.team_id,- mode: "search",- url: req.body.query,- scrapeOptions: req.body.scrapeOptions,- crawlerOptions: {- ...req.body,- query: undefined,- scrapeOptions: undefined,+ if (!isZDROrAnon) {+ logJob(+ {+ job_id: jobId,+ success: true,+ num_docs:+ (searchResponse.web?.length ?? 0) ++ (searchResponse.images?.length ?? 0) ++ (searchResponse.news?.length ?? 0),+ docs: [searchResponse],+ time_taken: timeTakenInSeconds,+ team_id: req.auth.team_id,+ mode: "search",+ url: req.body.query,+ scrapeOptions: req.body.scrapeOptions,+ crawlerOptions: {+ ...req.body,+ query: undefined,+ scrapeOptions: undefined,+ },+ origin: req.body.origin,+ integration: req.body.integration,+ credits_billed,+ zeroDataRetention: false,},- origin: req.body.origin,- integration: req.body.integration,- credits_billed,- zeroDataRetention: false,- },- false,- isSearchPreview,- );+ false,+ isSearchPreview,+ );+ }// Log final timing information for async modeconst totalRequestTime = new Date().getTime() - middlewareStartTime;@@ -588,7 +613,7 @@ export async function searchController({teamId: req.auth.team_id,bypassBilling: true,- zeroDataRetention: false,+ zeroDataRetention: isZDROrAnon,},docWithCost.document,docWithCost.costTracking,@@ -599,10 +624,16 @@ export async function searchController(try {const individualCredits = await Promise.all(creditPromises);- credits_billed = individualCredits.reduce(+ const scrapeCredits = individualCredits.reduce((sum, credit) => sum + credit,0,);++ const creditsPerTenResults = isZDR ? 10 : 2;+ const searchCredits =+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults;++ credits_billed = scrapeCredits + searchCredits;} catch (error) {logger.error("Error calculating credits for billing", { error });credits_billed = totalResultsCount;@@ -636,39 +667,40 @@ export async function searchController(const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;- logger.info("Logging job", {- num_docs: credits_billed,- time_taken: timeTakenInSeconds,- });-- logJob(- {- job_id: jobId,- success: true,- num_docs:- (searchResponse.web?.length ?? 0) +- (searchResponse.images?.length ?? 0) +- (searchResponse.news?.length ?? 0),- docs: [searchResponse],+ if (!isZDROrAnon) {+ logger.info("Logging job", {+ num_docs: credits_billed,time_taken: timeTakenInSeconds,- team_id: req.auth.team_id,- mode: "search",- url: req.body.query,- scrapeOptions: req.body.scrapeOptions,- crawlerOptions: {- ...req.body,- query: undefined,- scrapeOptions: undefined,- asyncScraping: isAsyncScraping,+ });+ logJob(+ {+ job_id: jobId,+ success: true,+ num_docs:+ (searchResponse.web?.length ?? 0) ++ (searchResponse.images?.length ?? 0) ++ (searchResponse.news?.length ?? 0),+ docs: [searchResponse],+ time_taken: timeTakenInSeconds,+ team_id: req.auth.team_id,+ mode: "search",+ url: req.body.query,+ scrapeOptions: req.body.scrapeOptions,+ crawlerOptions: {+ ...req.body,+ query: undefined,+ scrapeOptions: undefined,+ asyncScraping: isAsyncScraping,+ },+ origin: req.body.origin,+ integration: req.body.integration,+ credits_billed,+ zeroDataRetention: false, // not supported},- origin: req.body.origin,- integration: req.body.integration,- credits_billed,- zeroDataRetention: false, // not supported- },- false,- isSearchPreview,- );+ false,+ isSearchPreview,+ );+ }// Log final timing informationconst totalRequestTime = new Date().getTime() - middlewareStartTime;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 1599426c5..43e4a1a13 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1486,6 +1486,7 @@ export const searchRequestSchema = z),]).optional(),+ enterprise: z.array(z.enum(["default", "anon", "zdr"])).optional(),lang: z.string().optional().default("en"),country: z.string().optional(),location: z.string().optional(),diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 64b773c05..1ad45b71c 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -265,6 +265,7 @@ export async function x402SearchController(country: req.body.country,location: req.body.location,type: searchTypes,+ enterprise: req.body.enterprise,})) as SearchV2Response;// Add category labels to web resultsdiff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 564ac2448..89d97323e 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -42,6 +42,7 @@ export async function fire_engine_search_v2(numResults: number;page?: number;type?: SearchResultType | SearchResultType[];+ enterprise?: ("default" | "anon" | "zdr")[];},abort?: AbortSignal,): Promise<SearchV2Response> {@@ -61,6 +62,7 @@ export async function fire_engine_search_v2(numResults: options.numResults,page: options.page ?? 1,type: options.type || "web",+ enterprise: options.enterprise,};const requestedTypes = normalizeSearchTypes(options.type);diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex 5125c8c77..112ba5d09 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -18,6 +18,7 @@ export async function search({sleep_interval = 0,timeout = 5000,type = undefined,+ enterprise = undefined,}: {query: string;logger: Logger;@@ -32,6 +33,7 @@ export async function search({sleep_interval?: number;timeout?: number;type?: SearchResultType | SearchResultType[];+ enterprise?: ("default" | "anon" | "zdr")[];}): Promise<SearchV2Response> {try {if (process.env.FIRE_ENGINE_BETA_URL) {@@ -44,6 +46,7 @@ export async function search({country,location,type,+ enterprise,});return results;