Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(feat/search) Enterprise params (#2412)

Nicolasaf2db8d046bf
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 1ddf63799..1058fe359 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -391,10 +391,15 @@ export async function searchController(
try {
const individualCredits = await Promise.all(creditPromises);
- credits_billed = individualCredits.reduce(
+ const scrapeCredits = individualCredits.reduce(
(sum, credit) => sum + credit,
0,
);
+
+ // Add search credits (same as when no scrape options are specified)
+ const searchCredits = Math.ceil(responseData.data.length / 10) * 2;
+
+ credits_billed = scrapeCredits + searchCredits;
} catch (error) {
logger.error("Error calculating credits for billing", { error });
credits_billed = responseData.data.length;
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index c24c697fb..4b6534119 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -51,6 +51,7 @@ async function startScrapeJob(
scrapeOptions: ScrapeOptions;
bypassBilling?: boolean;
apiKeyId: number | null;
+ zeroDataRetention?: boolean;
},
logger: Logger,
flags: TeamFlags,
@@ -59,7 +60,8 @@ async function startScrapeJob(
): Promise<string> {
const jobId = uuidv7();
- const zeroDataRetention = flags?.forceZDR ?? false;
+ const zeroDataRetention =
+ options.zeroDataRetention ?? flags?.forceZDR ?? false;
logger.info("Adding scrape job", {
scrapeId: jobId,
@@ -114,6 +116,7 @@ async function scrapeSearchResult(
scrapeOptions: ScrapeOptions;
bypassBilling?: boolean;
apiKeyId: number | null;
+ zeroDataRetention?: boolean;
},
logger: Logger,
flags: TeamFlags,
@@ -250,6 +253,10 @@ export async function searchController(
// After transformation, sources is always an array of objects
const searchTypes = [...new Set(req.body.sources.map((s: any) => s.type))];
+ const isZDR = req.body.enterprise?.includes("zdr");
+ const isAnon = req.body.enterprise?.includes("anon");
+ const isZDROrAnon = isZDR || isAnon;
+
// Build search query with category filters
const { query: searchQuery, categoryMap } = buildSearchQuery(
req.body.query,
@@ -267,6 +274,7 @@ export async function searchController(
country: req.body.country,
location: req.body.location,
type: searchTypes,
+ enterprise: req.body.enterprise,
})) as SearchV2Response;
// Add category labels to web results
@@ -321,8 +329,8 @@ export async function searchController(
const isAsyncScraping = req.body.asyncScraping && shouldScrape;
if (!shouldScrape) {
- // No scraping - 2 credits per 10 search results
- credits_billed = Math.ceil(totalResultsCount / 10) * 2;
+ const creditsPerTenResults = isZDR ? 10 : 2;
+ credits_billed = Math.ceil(totalResultsCount / 10) * creditsPerTenResults;
} else {
// Common setup for both async and sync scraping
logger.info(
@@ -337,6 +345,7 @@ export async function searchController(
scrapeOptions: req.body.scrapeOptions,
bypassBilling: !isAsyncScraping, // Async mode bills per job, sync mode bills manually
apiKeyId: req.acuc?.api_key_id ?? null,
+ zeroDataRetention: isZDROrAnon,
};
const directToBullMQ = (req.acuc?.price_credits ?? 0) <= 3000;
@@ -472,8 +481,23 @@ export async function searchController(
);
}
- // Don't bill here - let each job bill itself when it completes
- credits_billed = allJobIds.length; // Just for reporting, not billing
+ const creditsPerTenResults = isZDR ? 10 : 2;
+ credits_billed =
+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults;
+
+ // Bill for search results now (scrape jobs will bill themselves when they complete)
+ if (!isSearchPreview) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ credits_billed,
+ req.acuc?.api_key_id ?? null,
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${req.acuc?.sub_id} for ${credits_billed} credits: ${error}`,
+ );
+ });
+ }
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
@@ -483,34 +507,35 @@ export async function searchController(
time_taken: timeTakenInSeconds,
scrapeIds,
});
-
- logJob(
- {
- job_id: jobId,
- success: true,
- num_docs:
- (searchResponse.web?.length ?? 0) +
- (searchResponse.images?.length ?? 0) +
- (searchResponse.news?.length ?? 0),
- docs: [searchResponse],
- time_taken: timeTakenInSeconds,
- team_id: req.auth.team_id,
- mode: "search",
- url: req.body.query,
- scrapeOptions: req.body.scrapeOptions,
- crawlerOptions: {
- ...req.body,
- query: undefined,
- scrapeOptions: undefined,
+ if (!isZDROrAnon) {
+ logJob(
+ {
+ job_id: jobId,
+ success: true,
+ num_docs:
+ (searchResponse.web?.length ?? 0) +
+ (searchResponse.images?.length ?? 0) +
+ (searchResponse.news?.length ?? 0),
+ docs: [searchResponse],
+ time_taken: timeTakenInSeconds,
+ team_id: req.auth.team_id,
+ mode: "search",
+ url: req.body.query,
+ scrapeOptions: req.body.scrapeOptions,
+ crawlerOptions: {
+ ...req.body,
+ query: undefined,
+ scrapeOptions: undefined,
+ },
+ origin: req.body.origin,
+ integration: req.body.integration,
+ credits_billed,
+ zeroDataRetention: false,
},
- origin: req.body.origin,
- integration: req.body.integration,
- credits_billed,
- zeroDataRetention: false,
- },
- false,
- isSearchPreview,
- );
+ false,
+ isSearchPreview,
+ );
+ }
// Log final timing information for async mode
const totalRequestTime = new Date().getTime() - middlewareStartTime;
@@ -588,7 +613,7 @@ export async function searchController(
{
teamId: req.auth.team_id,
bypassBilling: true,
- zeroDataRetention: false,
+ zeroDataRetention: isZDROrAnon,
},
docWithCost.document,
docWithCost.costTracking,
@@ -599,10 +624,16 @@ export async function searchController(
try {
const individualCredits = await Promise.all(creditPromises);
- credits_billed = individualCredits.reduce(
+ const scrapeCredits = individualCredits.reduce(
(sum, credit) => sum + credit,
0,
);
+
+ const creditsPerTenResults = isZDR ? 10 : 2;
+ const searchCredits =
+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults;
+
+ credits_billed = scrapeCredits + searchCredits;
} catch (error) {
logger.error("Error calculating credits for billing", { error });
credits_billed = totalResultsCount;
@@ -636,39 +667,40 @@ export async function searchController(
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
- logger.info("Logging job", {
- num_docs: credits_billed,
- time_taken: timeTakenInSeconds,
- });
-
- logJob(
- {
- job_id: jobId,
- success: true,
- num_docs:
- (searchResponse.web?.length ?? 0) +
- (searchResponse.images?.length ?? 0) +
- (searchResponse.news?.length ?? 0),
- docs: [searchResponse],
+ if (!isZDROrAnon) {
+ logger.info("Logging job", {
+ num_docs: credits_billed,
time_taken: timeTakenInSeconds,
- team_id: req.auth.team_id,
- mode: "search",
- url: req.body.query,
- scrapeOptions: req.body.scrapeOptions,
- crawlerOptions: {
- ...req.body,
- query: undefined,
- scrapeOptions: undefined,
- asyncScraping: isAsyncScraping,
+ });
+ logJob(
+ {
+ job_id: jobId,
+ success: true,
+ num_docs:
+ (searchResponse.web?.length ?? 0) +
+ (searchResponse.images?.length ?? 0) +
+ (searchResponse.news?.length ?? 0),
+ docs: [searchResponse],
+ time_taken: timeTakenInSeconds,
+ team_id: req.auth.team_id,
+ mode: "search",
+ url: req.body.query,
+ scrapeOptions: req.body.scrapeOptions,
+ crawlerOptions: {
+ ...req.body,
+ query: undefined,
+ scrapeOptions: undefined,
+ asyncScraping: isAsyncScraping,
+ },
+ origin: req.body.origin,
+ integration: req.body.integration,
+ credits_billed,
+ zeroDataRetention: false, // not supported
},
- origin: req.body.origin,
- integration: req.body.integration,
- credits_billed,
- zeroDataRetention: false, // not supported
- },
- false,
- isSearchPreview,
- );
+ false,
+ isSearchPreview,
+ );
+ }
// Log final timing information
const totalRequestTime = new Date().getTime() - middlewareStartTime;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 1599426c5..43e4a1a13 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1486,6 +1486,7 @@ export const searchRequestSchema = z
),
])
.optional(),
+ enterprise: z.array(z.enum(["default", "anon", "zdr"])).optional(),
lang: z.string().optional().default("en"),
country: z.string().optional(),
location: z.string().optional(),
diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.ts
index 64b773c05..1ad45b71c 100644
--- a/apps/api/src/controllers/v2/x402-search.ts
+++ b/apps/api/src/controllers/v2/x402-search.ts
@@ -265,6 +265,7 @@ export async function x402SearchController(
country: req.body.country,
location: req.body.location,
type: searchTypes,
+ enterprise: req.body.enterprise,
})) as SearchV2Response;
// Add category labels to web results
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 564ac2448..89d97323e 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -42,6 +42,7 @@ export async function fire_engine_search_v2(
numResults: number;
page?: number;
type?: SearchResultType | SearchResultType[];
+ enterprise?: ("default" | "anon" | "zdr")[];
},
abort?: AbortSignal,
): Promise<SearchV2Response> {
@@ -61,6 +62,7 @@ export async function fire_engine_search_v2(
numResults: options.numResults,
page: options.page ?? 1,
type: options.type || "web",
+ enterprise: options.enterprise,
};
const requestedTypes = normalizeSearchTypes(options.type);
diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.ts
index 5125c8c77..112ba5d09 100644
--- a/apps/api/src/search/v2/index.ts
+++ b/apps/api/src/search/v2/index.ts
@@ -18,6 +18,7 @@ export async function search({
sleep_interval = 0,
timeout = 5000,
type = undefined,
+ enterprise = undefined,
}: {
query: string;
logger: Logger;
@@ -32,6 +33,7 @@ export async function search({
sleep_interval?: number;
timeout?: number;
type?: SearchResultType | SearchResultType[];
+ enterprise?: ("default" | "anon" | "zdr")[];
}): Promise<SearchV2Response> {
try {
if (process.env.FIRE_ENGINE_BETA_URL) {
@@ -44,6 +46,7 @@ export async function search({
country,
location,
type,
+ enterprise,
});
return results;