Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: search v2 wip

Nicolasbb57be024fc6
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 82efbcfb3..adb60dfd4 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -7,7 +7,6 @@ import {
searchRequestSchema,
ScrapeOptions,
TeamFlags,
- scrapeOptions,
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
@@ -16,7 +15,7 @@ import { logJob } from "../../services/logging/log_job";
import { getJobPriority } from "../../lib/job-priority";
import { Mode } from "../../types";
import { getScrapeQueue } from "../../services/queue-service";
-import { search } from "../../search";
+import { search } from "../../search/v2";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
@@ -25,51 +24,13 @@ import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
+import { SearchResult, SearchV2Response } from "../../lib/entities";
interface DocumentWithCostTracking {
document: Document;
costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
}
-// Used for deep research
-export async function searchAndScrapeSearchResult(
- query: string,
- options: {
- teamId: string;
- origin: string;
- timeout: number;
- scrapeOptions: ScrapeOptions;
- },
- logger: Logger,
- flags: TeamFlags,
-): Promise<DocumentWithCostTracking[]> {
- try {
- const searchResults = await search({
- query,
- num_results: 5,
- });
-
- const documentsWithCostTracking = await Promise.all(
- searchResults.map((result) =>
- scrapeSearchResult(
- {
- url: result.url,
- title: result.title,
- description: result.description,
- },
- options,
- logger,
- flags,
- ),
- ),
- );
-
- return documentsWithCostTracking;
- } catch (error) {
- return [];
- }
-}
-
async function scrapeSearchResult(
searchResult: { url: string; title: string; description: string },
options: {
@@ -209,10 +170,6 @@ export async function searchController(
return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search. Please contact [email protected] to unblock this feature." });
}
- let responseData: SearchResponse = {
- success: true,
- data: [],
- };
const startTime = new Date().getTime();
const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
@@ -234,7 +191,7 @@ export async function searchController(
logger.info("Searching for results");
- let searchResults = await search({
+ const searchResponse = await search({
query: req.body.query,
advanced: false,
num_results: num_results_buffer,
@@ -243,105 +200,174 @@ export async function searchController(
lang: req.body.lang,
country: req.body.country,
location: req.body.location,
- });
+ type: req.body.type,
+ }) as SearchV2Response;
- if (req.body.ignoreInvalidURLs) {
- searchResults = searchResults.filter(
+ // Apply URL filtering if needed
+ if (req.body.ignoreInvalidURLs && searchResponse.web) {
+ searchResponse.web = searchResponse.web.filter(
(result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null),
);
}
- logger.info("Searching completed", {
- num_results: searchResults.length,
- });
-
- // Filter blocked URLs early to avoid unnecessary billing
- if (searchResults.length > limit) {
- searchResults = searchResults.slice(0, limit);
+ // Apply limit to each result type separately
+ let totalResultsCount = 0;
+
+ // Apply limit to web results
+ if (searchResponse.web && searchResponse.web.length > 0) {
+ if (searchResponse.web.length > limit) {
+ searchResponse.web = searchResponse.web.slice(0, limit);
+ }
+ totalResultsCount += searchResponse.web.length;
}
-
- if (searchResults.length === 0) {
- logger.info("No search results found");
- responseData.warning = "No search results found";
- } else if (
- !req.body.scrapeOptions.formats ||
- req.body.scrapeOptions.formats.length === 0
- ) {
- responseData.data = searchResults.map((r) => ({
- url: r.url,
- title: r.title,
- description: r.description,
- })) as Document[];
- credits_billed = responseData.data.length;
- } else {
- logger.info("Scraping search results");
- const scrapePromises = searchResults.map((result) =>
- scrapeSearchResult(
- result,
- {
- teamId: req.auth.team_id,
- origin: req.body.origin,
- timeout: req.body.timeout,
- scrapeOptions: req.body.scrapeOptions,
- },
- logger,
- req.acuc?.flags ?? null,
- (req.acuc?.price_credits ?? 0) <= 3000,
- isSearchPreview,
- ),
- );
-
- const docsWithCostTracking = await Promise.all(scrapePromises);
- logger.info("Scraping completed", {
- num_docs: docsWithCostTracking.length,
- });
-
- const docs = docsWithCostTracking.map(item => item.document);
- const filteredDocs = docs.filter(
- (doc) =>
- doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),
- );
-
- logger.info("Filtering completed", {
- num_docs: filteredDocs.length,
- });
-
- if (filteredDocs.length === 0) {
- responseData.data = docs;
- responseData.warning = "No content found in search results";
- } else {
- responseData.data = filteredDocs;
+
+ // Apply limit to images
+ if (searchResponse.images && searchResponse.images.length > 0) {
+ if (searchResponse.images.length > limit) {
+ searchResponse.images = searchResponse.images.slice(0, limit);
}
-
- const finalDocsForBilling = responseData.data;
+ totalResultsCount += searchResponse.images.length;
+ }
+
+ // Apply limit to news
+ if (searchResponse.news && searchResponse.news.length > 0) {
+ if (searchResponse.news.length > limit) {
+ searchResponse.news = searchResponse.news.slice(0, limit);
+ }
+ totalResultsCount += searchResponse.news.length;
+ }
+
+ // Check if scraping is requested
+ const shouldScrape = req.body.scrapeOptions.formats && req.body.scrapeOptions.formats.length > 0;
+
+ if (!shouldScrape) {
+ // No scraping - just count results for billing
+ credits_billed = totalResultsCount;
+ } else {
+ // Scrape web and news results (images don't need scraping)
+ const scrapedResponse: SearchV2Response = {};
- const creditPromises = finalDocsForBilling.map(async (finalDoc) => {
- const matchingDocWithCost = docsWithCostTracking.find(item =>
- item.document.metadata && finalDoc.metadata && item.document.metadata.scrapeId === finalDoc.metadata.scrapeId
+ // Handle web results
+ if (searchResponse.web && searchResponse.web.length > 0) {
+ logger.info("Scraping web search results");
+ const webSearchResults = searchResponse.web.map(item =>
+ new SearchResult(item.url, item.title, item.description)
);
- if (matchingDocWithCost) {
- return await calculateCreditsToBeBilled(
- req.body.scrapeOptions,
- { teamId: req.auth.team_id, bypassBilling: true, zeroDataRetention: false },
- matchingDocWithCost.document,
- matchingDocWithCost.costTracking,
+ const webScrapePromises = webSearchResults.map((result) =>
+ scrapeSearchResult(
+ {
+ url: result.url,
+ title: result.title,
+ description: result.description,
+ },
+ {
+ teamId: req.auth.team_id,
+ origin: req.body.origin,
+ timeout: req.body.timeout,
+ scrapeOptions: req.body.scrapeOptions,
+ },
+ logger,
req.acuc?.flags ?? null,
+ (req.acuc?.price_credits ?? 0) <= 3000,
+ isSearchPreview,
+ ),
+ );
+
+ const webDocsWithCostTracking = await Promise.all(webScrapePromises);
+ const webDocs = webDocsWithCostTracking.map(item => item.document);
+
+ // Convert scraped documents back to web search results with scraped content
+ scrapedResponse.web = webDocs.map((doc, index) => ({
+ url: doc.url || searchResponse.web![index].url,
+ title: doc.title || searchResponse.web![index].title,
+ description: doc.description || searchResponse.web![index].description,
+ position: searchResponse.web![index].position,
+ markdown: doc.markdown,
+ html: doc.html,
+ rawHtml: doc.rawHtml,
+ links: doc.links,
+ screenshot: doc.screenshot,
+ metadata: doc.metadata,
+ }));
+
+ allDocsWithCostTracking.push(...webDocsWithCostTracking);
+ }
+
+ // Handle news results
+ if (searchResponse.news && searchResponse.news.length > 0) {
+ logger.info("Scraping news search results");
+ const newsScrapePromises = searchResponse.news
+ .filter(item => item.url) // Only scrape news items with links
+ .map((newsItem) =>
+ scrapeSearchResult(
+ {
+ url: newsItem.url!,
+ title: newsItem.title || "",
+ description: newsItem.snippet || "",
+ },
+ {
+ teamId: req.auth.team_id,
+ origin: req.body.origin,
+ timeout: req.body.timeout,
+ scrapeOptions: req.body.scrapeOptions,
+ },
+ logger,
+ req.acuc?.flags ?? null,
+ (req.acuc?.price_credits ?? 0) <= 3000,
+ isSearchPreview,
+ ),
);
- } else {
- return 1;
- }
+
+ const newsDocsWithCostTracking = await Promise.all(newsScrapePromises);
+ const newsDocs = newsDocsWithCostTracking.map(item => item.document);
+
+ // Convert scraped documents back to news search results with scraped content
+ scrapedResponse.news = searchResponse.news.map((newsItem, index) => {
+ const scrapedDoc = newsDocs.find(doc => doc.url === newsItem.url);
+ return {
+ ...newsItem,
+ markdown: scrapedDoc?.markdown,
+ html: scrapedDoc?.html,
+ rawHtml: scrapedDoc?.rawHtml,
+ metadata: scrapedDoc?.metadata,
+ };
+ });
+
+ allDocsWithCostTracking.push(...newsDocsWithCostTracking);
+ }
+
+ // Include images without scraping
+ if (searchResponse.images) {
+ scrapedResponse.images = searchResponse.images;
+ }
+
+ // Calculate credits
+ const creditPromises = allDocsWithCostTracking.map(async (docWithCost) => {
+ return await calculateCreditsToBeBilled(
+ req.body.scrapeOptions,
+ { teamId: req.auth.team_id, bypassBilling: true, zeroDataRetention: false },
+ docWithCost.document,
+ docWithCost.costTracking,
+ req.acuc?.flags ?? null,
+ );
});
try {
const individualCredits = await Promise.all(creditPromises);
credits_billed = individualCredits.reduce((sum, credit) => sum + credit, 0);
+ // Add 1 credit per image result
+ if (scrapedResponse.images) {
+ credits_billed += scrapedResponse.images.length;
+ }
} catch (error) {
logger.error("Error calculating credits for billing", { error });
- credits_billed = responseData.data.length;
+ // Use the totalResultsCount which already respects the limit
+ credits_billed = totalResultsCount;
}
-
- allDocsWithCostTracking = docsWithCostTracking;
+
+ // Update response with scraped data
+ Object.assign(searchResponse, scrapedResponse);
}
// Bill team once for all successful results
@@ -352,7 +378,7 @@ export async function searchController(
credits_billed,
).catch((error) => {
logger.error(
- `Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,
+ `Failed to bill team ${req.auth.team_id} for ${credits_billed} credits: ${error}`,
);
});
}
@@ -361,7 +387,7 @@ export async function searchController(
const timeTakenInSeconds = (endTime - startTime) / 1000;
logger.info("Logging job", {
- num_docs: responseData.data.length,
+ num_docs: credits_billed,
time_taken: timeTakenInSeconds,
});
@@ -369,8 +395,8 @@ export async function searchController(
{
job_id: jobId,
success: true,
- num_docs: responseData.data.length,
- docs: responseData.data,
+ num_docs: credits_billed,
+ docs: [searchResponse],
time_taken: timeTakenInSeconds,
team_id: req.auth.team_id,
mode: "search",
@@ -390,7 +416,10 @@ export async function searchController(
isSearchPreview,
);
- return res.status(200).json(responseData);
+ return res.status(200).json({
+ success: true,
+ data: searchResponse,
+ });
} catch (error) {
if (
error instanceof Error &&
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 84ca8d6e7..ea919a21e 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1296,6 +1296,10 @@ export const searchRequestSchema = z
.default(5),
tbs: z.string().optional(),
filter: z.string().optional(),
+ type: z.union([
+ z.enum(["web", "images", "news"]),
+ z.array(z.enum(["web", "images", "news"]))
+ ]).optional().default("web"),
lang: z.string().optional().default("en"),
country: z.string().optional().default("us"),
location: z.string().optional(),
@@ -1343,6 +1347,11 @@ export type SearchResponse =
success: true;
warning?: string;
data: Document[];
+ }
+ | {
+ success: true;
+ warning?: string;
+ data: import("../../lib/entities").SearchV2Response;
};
export type TokenUsage = {
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 30aa0430a..e1f15c1cb 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -151,6 +151,53 @@ export class SearchResult {
}
}
+export interface ImageSearchResult {
+ title?: string;
+ imageUrl?: string;
+ imageWidth?: number;
+ imageHeight?: number;
+ url?: string;
+ position?: number;
+}
+
+export interface NewsSearchResult {
+ title?: string;
+ url?: string;
+ snippet?: string;
+ date?: string;
+ imageUrl?: string;
+ position?: number;
+ // Scraped content fields
+ markdown?: string;
+ html?: string;
+ rawHtml?: string;
+ links?: string[];
+ screenshot?: string;
+ metadata?: Record<string, any>;
+}
+
+export interface WebSearchResult {
+ url: string;
+ title: string;
+ description: string;
+ position?: number;
+ // Scraped content fields
+ markdown?: string;
+ html?: string;
+ rawHtml?: string;
+ links?: string[];
+ screenshot?: string;
+ metadata?: Record<string, any>;
+}
+
+export type SearchResultType = 'web' | 'images' | 'news';
+
+export interface SearchV2Response {
+ web?: WebSearchResult[];
+ images?: ImageSearchResult[];
+ news?: NewsSearchResult[];
+}
+
export interface ScrapeActionContent {
url: string;
html: string;
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 75d398f9d..1122e82ac 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -1,7 +1,7 @@
import express from "express";
import { RateLimiterMode } from "../types";
import expressWs from "express-ws";
-import { searchController } from "../controllers/v1/search";
+import { searchController } from "../controllers/v2/search";
import {
authMiddleware,
checkCreditsMiddleware,
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 69c0c1197..3f606c63b 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -1,12 +1,11 @@
import dotenv from "dotenv";
-import { SearchResult } from "../../lib/entities";
+import { SearchResult, SearchV2Response, SearchResultType } from "../../lib/entities";
import * as Sentry from "@sentry/node";
import { logger } from "../../lib/logger";
dotenv.config();
-
-export async function fire_engine_search(
+export async function fire_engine_search_v2(
q: string,
options: {
tbs?: string;
@@ -16,9 +15,10 @@ export async function fire_engine_search(
location?: string;
numResults: number;
page?: number;
+ type?: SearchResultType | SearchResultType[];
},
abort?: AbortSignal,
-): Promise<SearchResult[]> {
+): Promise<SearchV2Response> {
try {
let data = JSON.stringify({
query: q,
@@ -28,10 +28,11 @@ export async function fire_engine_search(
tbs: options.tbs,
numResults: options.numResults,
page: options.page ?? 1,
+ type: options.type || 'web',
});
if (!process.env.FIRE_ENGINE_BETA_URL) {
- return [];
+ return {};
}
const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/v2/search`, {
@@ -48,13 +49,40 @@ export async function fire_engine_search(
const responseData = await response.json();
return responseData;
} else {
- return [];
+ return {};
}
} catch (error) {
logger.error(error);
Sentry.captureException(error);
- return [];
+ return {};
+ }
+}
+
+// Backward compatibility function
+export async function fire_engine_search(
+ q: string,
+ options: {
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ numResults: number;
+ page?: number;
+ type?: SearchResultType | SearchResultType[];
+ },
+ abort?: AbortSignal,
+): Promise<SearchResult[]> {
+ const result = await fire_engine_search_v2(q, options, abort);
+
+ // Handle backward compatibility - convert to SearchResult array
+ if (Array.isArray(result)) {
+ return result;
+ } else if (result.web) {
+ return result.web.map(item => new SearchResult(item.url, item.title, item.description));
}
+
+ return [];
}
export async function fireEngineMap(
diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.ts
new file mode 100644
index 000000000..38cef0f1f
--- /dev/null
+++ b/apps/api/src/search/v2/index.ts
@@ -0,0 +1,99 @@
+import { logger } from "../../lib/logger";
+import { SearchResult, SearchV2Response, SearchResultType } from "../../lib/entities";
+import { fire_engine_search, fire_engine_search_v2 } from "./fireEngine-v2";
+
+export async function search({
+ query,
+ advanced = false,
+ num_results = 5,
+ tbs = undefined,
+ filter = undefined,
+ lang = "en",
+ country = "us",
+ location = undefined,
+ proxy = undefined,
+ sleep_interval = 0,
+ timeout = 5000,
+ type = undefined,
+}: {
+ query: string;
+ advanced?: boolean;
+ num_results?: number;
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ proxy?: string;
+ sleep_interval?: number;
+ timeout?: number;
+ type?: SearchResultType | SearchResultType[];
+}): Promise<SearchV2Response> {
+ try {
+ if (process.env.FIRE_ENGINE_BETA_URL) {
+ const results = await fire_engine_search_v2(query, {
+ numResults: num_results,
+ tbs,
+ filter,
+ lang,
+ country,
+ location,
+ type,
+ });
+
+ return results;
+ }
+
+ // Fallback to empty response
+ return {};
+
+ // if (process.env.SERPER_API_KEY) {
+ // const results = await serper_search(query, {
+ // num_results,
+ // tbs,
+ // filter,
+ // lang,
+ // country,
+ // location,
+ // });
+ // if (results.length > 0) return results;
+ // }
+ // if (process.env.SEARCHAPI_API_KEY) {
+ // const results = await searchapi_search(query, {
+ // num_results,
+ // tbs,
+ // filter,
+ // lang,
+ // country,
+ // location,
+ // });
+ // if (results.length > 0) return results;
+ // }
+ // if (process.env.SEARXNG_ENDPOINT) {
+ // const results = await searxng_search(query, {
+ // num_results,
+ // tbs,
+ // filter,
+ // lang,
+ // country,
+ // location,
+ // });
+ // if (results.length > 0) return results;
+ // }
+ // return await googleSearch(
+ // query,
+ // advanced,
+ // num_results,
+ // tbs,
+ // filter,
+ // lang,
+ // country,
+ // proxy,
+ // sleep_interval,
+ // timeout,
+ // );
+ } catch (error) {
+ logger.error(`Error in search function`, { error });
+ return {};
+ }
+}