Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: search v2 wip
Nicolasa9fa2f0aa5f9
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 82efbcfb3..adb60dfd4 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -7,7 +7,6 @@ import {searchRequestSchema,ScrapeOptions,TeamFlags,- scrapeOptions,} from "./types";import { billTeam } from "../../services/billing/credit_billing";import { v4 as uuidv4 } from "uuid";@@ -16,7 +15,7 @@ import { logJob } from "../../services/logging/log_job";import { getJobPriority } from "../../lib/job-priority";import { Mode } from "../../types";import { getScrapeQueue } from "../../services/queue-service";-import { search } from "../../search";+import { search } from "../../search/v2";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";@@ -25,51 +24,13 @@ import type { Logger } from "winston";import { CostTracking } from "../../lib/extract/extraction-service";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";+import { SearchResult, SearchV2Response } from "../../lib/entities";interface DocumentWithCostTracking {document: Document;costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;}-// Used for deep research-export async function searchAndScrapeSearchResult(- query: string,- options: {- teamId: string;- origin: string;- timeout: number;- scrapeOptions: ScrapeOptions;- },- logger: Logger,- flags: TeamFlags,-): Promise<DocumentWithCostTracking[]> {- try {- const searchResults = await search({- query,- num_results: 5,- });-- const documentsWithCostTracking = await Promise.all(- searchResults.map((result) =>- scrapeSearchResult(- {- url: result.url,- title: result.title,- description: result.description,- },- options,- logger,- flags,- ),- ),- );-- return documentsWithCostTracking;- } catch (error) {- return [];- }-}-async function scrapeSearchResult(searchResult: { url: string; title: string; description: string },options: {@@ -209,10 +170,6 @@ export async function searchController(return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search. Please contact [email protected] to unblock this feature." });}- let responseData: SearchResponse = {- success: true,- data: [],- };const startTime = new Date().getTime();const isSearchPreview = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;@@ -234,7 +191,7 @@ export async function searchController(logger.info("Searching for results");- let searchResults = await search({+ const searchResponse = await search({query: req.body.query,advanced: false,num_results: num_results_buffer,@@ -243,105 +200,174 @@ export async function searchController(lang: req.body.lang,country: req.body.country,location: req.body.location,- });+ type: req.body.type,+ }) as SearchV2Response;- if (req.body.ignoreInvalidURLs) {- searchResults = searchResults.filter(+ // Apply URL filtering if needed+ if (req.body.ignoreInvalidURLs && searchResponse.web) {+ searchResponse.web = searchResponse.web.filter((result) => !isUrlBlocked(result.url, req.acuc?.flags ?? null),);}- logger.info("Searching completed", {- num_results: searchResults.length,- });-- // Filter blocked URLs early to avoid unnecessary billing- if (searchResults.length > limit) {- searchResults = searchResults.slice(0, limit);+ // Apply limit to each result type separately+ let totalResultsCount = 0;++ // Apply limit to web results+ if (searchResponse.web && searchResponse.web.length > 0) {+ if (searchResponse.web.length > limit) {+ searchResponse.web = searchResponse.web.slice(0, limit);+ }+ totalResultsCount += searchResponse.web.length;}-- if (searchResults.length === 0) {- logger.info("No search results found");- responseData.warning = "No search results found";- } else if (- !req.body.scrapeOptions.formats ||- req.body.scrapeOptions.formats.length === 0- ) {- responseData.data = searchResults.map((r) => ({- url: r.url,- title: r.title,- description: r.description,- })) as Document[];- credits_billed = responseData.data.length;- } else {- logger.info("Scraping search results");- const scrapePromises = searchResults.map((result) =>- scrapeSearchResult(- result,- {- teamId: req.auth.team_id,- origin: req.body.origin,- timeout: req.body.timeout,- scrapeOptions: req.body.scrapeOptions,- },- logger,- req.acuc?.flags ?? null,- (req.acuc?.price_credits ?? 0) <= 3000,- isSearchPreview,- ),- );-- const docsWithCostTracking = await Promise.all(scrapePromises);- logger.info("Scraping completed", {- num_docs: docsWithCostTracking.length,- });-- const docs = docsWithCostTracking.map(item => item.document);- const filteredDocs = docs.filter(- (doc) =>- doc.serpResults || (doc.markdown && doc.markdown.trim().length > 0),- );-- logger.info("Filtering completed", {- num_docs: filteredDocs.length,- });-- if (filteredDocs.length === 0) {- responseData.data = docs;- responseData.warning = "No content found in search results";- } else {- responseData.data = filteredDocs;++ // Apply limit to images+ if (searchResponse.images && searchResponse.images.length > 0) {+ if (searchResponse.images.length > limit) {+ searchResponse.images = searchResponse.images.slice(0, limit);}-- const finalDocsForBilling = responseData.data;+ totalResultsCount += searchResponse.images.length;+ }++ // Apply limit to news+ if (searchResponse.news && searchResponse.news.length > 0) {+ if (searchResponse.news.length > limit) {+ searchResponse.news = searchResponse.news.slice(0, limit);+ }+ totalResultsCount += searchResponse.news.length;+ }++ // Check if scraping is requested+ const shouldScrape = req.body.scrapeOptions.formats && req.body.scrapeOptions.formats.length > 0;++ if (!shouldScrape) {+ // No scraping - just count results for billing+ credits_billed = totalResultsCount;+ } else {+ // Scrape web and news results (images don't need scraping)+ const scrapedResponse: SearchV2Response = {};- const creditPromises = finalDocsForBilling.map(async (finalDoc) => {- const matchingDocWithCost = docsWithCostTracking.find(item =>- item.document.metadata && finalDoc.metadata && item.document.metadata.scrapeId === finalDoc.metadata.scrapeId+ // Handle web results+ if (searchResponse.web && searchResponse.web.length > 0) {+ logger.info("Scraping web search results");+ const webSearchResults = searchResponse.web.map(item =>+ new SearchResult(item.url, item.title, item.description));- if (matchingDocWithCost) {- return await calculateCreditsToBeBilled(- req.body.scrapeOptions,- { teamId: req.auth.team_id, bypassBilling: true, zeroDataRetention: false },- matchingDocWithCost.document,- matchingDocWithCost.costTracking,+ const webScrapePromises = webSearchResults.map((result) =>+ scrapeSearchResult(+ {+ url: result.url,+ title: result.title,+ description: result.description,+ },+ {+ teamId: req.auth.team_id,+ origin: req.body.origin,+ timeout: req.body.timeout,+ scrapeOptions: req.body.scrapeOptions,+ },+ logger,req.acuc?.flags ?? null,+ (req.acuc?.price_credits ?? 0) <= 3000,+ isSearchPreview,+ ),+ );++ const webDocsWithCostTracking = await Promise.all(webScrapePromises);+ const webDocs = webDocsWithCostTracking.map(item => item.document);++ // Convert scraped documents back to web search results with scraped content+ scrapedResponse.web = webDocs.map((doc, index) => ({+ url: doc.url || searchResponse.web![index].url,+ title: doc.title || searchResponse.web![index].title,+ description: doc.description || searchResponse.web![index].description,+ position: searchResponse.web![index].position,+ markdown: doc.markdown,+ html: doc.html,+ rawHtml: doc.rawHtml,+ links: doc.links,+ screenshot: doc.screenshot,+ metadata: doc.metadata,+ }));++ allDocsWithCostTracking.push(...webDocsWithCostTracking);+ }++ // Handle news results+ if (searchResponse.news && searchResponse.news.length > 0) {+ logger.info("Scraping news search results");+ const newsScrapePromises = searchResponse.news+ .filter(item => item.url) // Only scrape news items with links+ .map((newsItem) =>+ scrapeSearchResult(+ {+ url: newsItem.url!,+ title: newsItem.title || "",+ description: newsItem.snippet || "",+ },+ {+ teamId: req.auth.team_id,+ origin: req.body.origin,+ timeout: req.body.timeout,+ scrapeOptions: req.body.scrapeOptions,+ },+ logger,+ req.acuc?.flags ?? null,+ (req.acuc?.price_credits ?? 0) <= 3000,+ isSearchPreview,+ ),);- } else {- return 1;- }++ const newsDocsWithCostTracking = await Promise.all(newsScrapePromises);+ const newsDocs = newsDocsWithCostTracking.map(item => item.document);++ // Convert scraped documents back to news search results with scraped content+ scrapedResponse.news = searchResponse.news.map((newsItem, index) => {+ const scrapedDoc = newsDocs.find(doc => doc.url === newsItem.url);+ return {+ ...newsItem,+ markdown: scrapedDoc?.markdown,+ html: scrapedDoc?.html,+ rawHtml: scrapedDoc?.rawHtml,+ metadata: scrapedDoc?.metadata,+ };+ });++ allDocsWithCostTracking.push(...newsDocsWithCostTracking);+ }++ // Include images without scraping+ if (searchResponse.images) {+ scrapedResponse.images = searchResponse.images;+ }++ // Calculate credits+ const creditPromises = allDocsWithCostTracking.map(async (docWithCost) => {+ return await calculateCreditsToBeBilled(+ req.body.scrapeOptions,+ { teamId: req.auth.team_id, bypassBilling: true, zeroDataRetention: false },+ docWithCost.document,+ docWithCost.costTracking,+ req.acuc?.flags ?? null,+ );});try {const individualCredits = await Promise.all(creditPromises);credits_billed = individualCredits.reduce((sum, credit) => sum + credit, 0);+ // Add 1 credit per image result+ if (scrapedResponse.images) {+ credits_billed += scrapedResponse.images.length;+ }} catch (error) {logger.error("Error calculating credits for billing", { error });- credits_billed = responseData.data.length;+ // Use the totalResultsCount which already respects the limit+ credits_billed = totalResultsCount;}-- allDocsWithCostTracking = docsWithCostTracking;++ // Update response with scraped data+ Object.assign(searchResponse, scrapedResponse);}// Bill team once for all successful results@@ -352,7 +378,7 @@ export async function searchController(credits_billed,).catch((error) => {logger.error(- `Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,+ `Failed to bill team ${req.auth.team_id} for ${credits_billed} credits: ${error}`,);});}@@ -361,7 +387,7 @@ export async function searchController(const timeTakenInSeconds = (endTime - startTime) / 1000;logger.info("Logging job", {- num_docs: responseData.data.length,+ num_docs: credits_billed,time_taken: timeTakenInSeconds,});@@ -369,8 +395,8 @@ export async function searchController({job_id: jobId,success: true,- num_docs: responseData.data.length,- docs: responseData.data,+ num_docs: credits_billed,+ docs: [searchResponse],time_taken: timeTakenInSeconds,team_id: req.auth.team_id,mode: "search",@@ -390,7 +416,10 @@ export async function searchController(isSearchPreview,);- return res.status(200).json(responseData);+ return res.status(200).json({+ success: true,+ data: searchResponse,+ });} catch (error) {if (error instanceof Error &&diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 84ca8d6e7..ea919a21e 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1296,6 +1296,10 @@ export const searchRequestSchema = z.default(5),tbs: z.string().optional(),filter: z.string().optional(),+ type: z.union([+ z.enum(["web", "images", "news"]),+ z.array(z.enum(["web", "images", "news"]))+ ]).optional().default("web"),lang: z.string().optional().default("en"),country: z.string().optional().default("us"),location: z.string().optional(),@@ -1343,6 +1347,11 @@ export type SearchResponse =success: true;warning?: string;data: Document[];+ }+ | {+ success: true;+ warning?: string;+ data: import("../../lib/entities").SearchV2Response;};export type TokenUsage = {diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 30aa0430a..e1f15c1cb 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -151,6 +151,53 @@ export class SearchResult {}}+export interface ImageSearchResult {+ title?: string;+ imageUrl?: string;+ imageWidth?: number;+ imageHeight?: number;+ url?: string;+ position?: number;+}++export interface NewsSearchResult {+ title?: string;+ url?: string;+ snippet?: string;+ date?: string;+ imageUrl?: string;+ position?: number;+ // Scraped content fields+ markdown?: string;+ html?: string;+ rawHtml?: string;+ links?: string[];+ screenshot?: string;+ metadata?: Record<string, any>;+}++export interface WebSearchResult {+ url: string;+ title: string;+ description: string;+ position?: number;+ // Scraped content fields+ markdown?: string;+ html?: string;+ rawHtml?: string;+ links?: string[];+ screenshot?: string;+ metadata?: Record<string, any>;+}++export type SearchResultType = 'web' | 'images' | 'news';++export interface SearchV2Response {+ web?: WebSearchResult[];+ images?: ImageSearchResult[];+ news?: NewsSearchResult[];+}+export interface ScrapeActionContent {url: string;html: string;diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 75d398f9d..1122e82ac 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -1,7 +1,7 @@import express from "express";import { RateLimiterMode } from "../types";import expressWs from "express-ws";-import { searchController } from "../controllers/v1/search";+import { searchController } from "../controllers/v2/search";import {authMiddleware,checkCreditsMiddleware,diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 69c0c1197..3f606c63b 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -1,12 +1,11 @@import dotenv from "dotenv";-import { SearchResult } from "../../lib/entities";+import { SearchResult, SearchV2Response, SearchResultType } from "../../lib/entities";import * as Sentry from "@sentry/node";import { logger } from "../../lib/logger";dotenv.config();--export async function fire_engine_search(+export async function fire_engine_search_v2(q: string,options: {tbs?: string;@@ -16,9 +15,10 @@ export async function fire_engine_search(location?: string;numResults: number;page?: number;+ type?: SearchResultType | SearchResultType[];},abort?: AbortSignal,-): Promise<SearchResult[]> {+): Promise<SearchV2Response> {try {let data = JSON.stringify({query: q,@@ -28,10 +28,11 @@ export async function fire_engine_search(tbs: options.tbs,numResults: options.numResults,page: options.page ?? 1,+ type: options.type || 'web',});if (!process.env.FIRE_ENGINE_BETA_URL) {- return [];+ return {};}const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/v2/search`, {@@ -48,13 +49,40 @@ export async function fire_engine_search(const responseData = await response.json();return responseData;} else {- return [];+ return {};}} catch (error) {logger.error(error);Sentry.captureException(error);- return [];+ return {};+ }+}++// Backward compatibility function+export async function fire_engine_search(+ q: string,+ options: {+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ numResults: number;+ page?: number;+ type?: SearchResultType | SearchResultType[];+ },+ abort?: AbortSignal,+): Promise<SearchResult[]> {+ const result = await fire_engine_search_v2(q, options, abort);++ // Handle backward compatibility - convert to SearchResult array+ if (Array.isArray(result)) {+ return result;+ } else if (result.web) {+ return result.web.map(item => new SearchResult(item.url, item.title, item.description));}++ return [];}export async function fireEngineMap(diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsnew file mode 100644index 000000000..38cef0f1f--- /dev/null+++ b/apps/api/src/search/v2/index.ts@@ -0,0 +1,99 @@+import { logger } from "../../lib/logger";+import { SearchResult, SearchV2Response, SearchResultType } from "../../lib/entities";+import { fire_engine_search, fire_engine_search_v2 } from "./fireEngine-v2";++export async function search({+ query,+ advanced = false,+ num_results = 5,+ tbs = undefined,+ filter = undefined,+ lang = "en",+ country = "us",+ location = undefined,+ proxy = undefined,+ sleep_interval = 0,+ timeout = 5000,+ type = undefined,+}: {+ query: string;+ advanced?: boolean;+ num_results?: number;+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ proxy?: string;+ sleep_interval?: number;+ timeout?: number;+ type?: SearchResultType | SearchResultType[];+}): Promise<SearchV2Response> {+ try {+ if (process.env.FIRE_ENGINE_BETA_URL) {+ const results = await fire_engine_search_v2(query, {+ numResults: num_results,+ tbs,+ filter,+ lang,+ country,+ location,+ type,+ });++ return results;+ }++ // Fallback to empty response+ return {};++ // if (process.env.SERPER_API_KEY) {+ // const results = await serper_search(query, {+ // num_results,+ // tbs,+ // filter,+ // lang,+ // country,+ // location,+ // });+ // if (results.length > 0) return results;+ // }+ // if (process.env.SEARCHAPI_API_KEY) {+ // const results = await searchapi_search(query, {+ // num_results,+ // tbs,+ // filter,+ // lang,+ // country,+ // location,+ // });+ // if (results.length > 0) return results;+ // }+ // if (process.env.SEARXNG_ENDPOINT) {+ // const results = await searxng_search(query, {+ // num_results,+ // tbs,+ // filter,+ // lang,+ // country,+ // location,+ // });+ // if (results.length > 0) return results;+ // }+ // return await googleSearch(+ // query,+ // advanced,+ // num_results,+ // tbs,+ // filter,+ // lang,+ // country,+ // proxy,+ // sleep_interval,+ // timeout,+ // );+ } catch (error) {+ logger.error(`Error in search function`, { error });+ return {};+ }+}