Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix search
Gergő Móriczd3bda3f011a6
diff --git a/apps/api/src/search/v2/googlesearch.ts b/apps/api/src/search/v2/googlesearch.tsnew file mode 100644index 000000000..f705a5430--- /dev/null+++ b/apps/api/src/search/v2/googlesearch.ts@@ -0,0 +1,169 @@+import axios from "axios";+import { JSDOM } from 'jsdom';+import { SearchV2Response, WebSearchResult } from "../../lib/entities";+import { logger } from "../../lib/logger";+import https from 'https';++const getRandomInt = (min: number, max: number): number => Math.floor(Math.random() * (max - min + 1)) + min;++export function get_useragent(): string {+ const lynx_version = `Lynx/${getRandomInt(2, 3)}.${getRandomInt(8, 9)}.${getRandomInt(0, 2)}`;+ const libwww_version = `libwww-FM/${getRandomInt(2, 3)}.${getRandomInt(13, 15)}`;+ const ssl_mm_version = `SSL-MM/${getRandomInt(1, 2)}.${getRandomInt(3, 5)}`;+ const openssl_version = `OpenSSL/${getRandomInt(1, 3)}.${getRandomInt(0, 4)}.${getRandomInt(0, 9)}`;+ return `${lynx_version} ${libwww_version} ${ssl_mm_version} ${openssl_version}`;+}++async function _req(+ term: string,+ results: number,+ lang: string,+ country: string,+ start: number,+ proxies: any,+ timeout: number,+ tbs: string | undefined = undefined,+ filter: string | undefined = undefined,+) {+ const params = {+ q: term,+ num: results+2, // Number of results to return+ hl: lang,+ gl: country,+ safe: "active",+ start: start,+ };+ if (tbs) {+ params["tbs"] = tbs;+ }+ if (filter) {+ params["filter"] = filter;+ }+ var agent = get_useragent();+ try {+ const resp = await axios.get("https://www.google.com/search", {+ headers: {+ "User-Agent": agent,+ "Accept": "*/*"+ },+ params: params,+ proxy: proxies,+ timeout: timeout,+ httpsAgent: new https.Agent({+ rejectUnauthorized: true+ }),+ withCredentials: true+ });+ return resp;+ } catch (error) {+ if (error.response && error.response.status === 429) {+ logger.warn("Google Search: Too many requests, try again later.", {+ status: error.response.status,+ statusText: error.response.statusText+ });+ throw new Error("Google Search: Too many requests, try again later.");+ }+ throw error;+ }+}++export async function googleSearch(+ term: string,+ advanced = false,+ num_results = 5,+ tbs = undefined as string | undefined,+ filter = undefined as string | undefined,+ lang = "en",+ country = "us",+ proxy = undefined as string | undefined,+ sleep_interval = 0,+ timeout = 5000,+): Promise<SearchV2Response> {+ let proxies: any = null;+ if (proxy) {+ if (proxy.startsWith("https")) {+ proxies = { https: proxy };+ } else {+ proxies = { http: proxy };+ }+ }++ // TODO: knowledge graph, answer box, etc.++ let start = 0;+ let results: WebSearchResult[] = [];+ let attempts = 0;+ const maxAttempts = 20; // Define a maximum number of attempts to prevent infinite loop+ while (start < num_results && attempts < maxAttempts) {+ try {+ const resp = await _req(+ term,+ num_results - start,+ lang,+ country,+ start,+ proxies,+ timeout,+ tbs,+ filter,+ );+ const dom = new JSDOM(resp.data);+ const document = dom.window.document;+ const result_block = document.querySelectorAll("div.ezO2md");+ let new_results = 0;+ let unique = true;+ let fetched_results = 0;++ const fetched_links = new Set<string>();+ if (result_block.length === 0) {+ start += 1;+ attempts += 1;+ } else {+ attempts = 0;+ }++ for (const result of result_block) {+ const link_tag = result.querySelector("a[href]") as HTMLAnchorElement;+ const title_tag = link_tag ? link_tag.querySelector("span.CVA68e") : null;+ const description_tag = result.querySelector("span.FrIlee");++ if (link_tag && title_tag && description_tag) {+ const link = decodeURIComponent(link_tag.href.split("&")[0].replace("/url?q=", ""));+ if (fetched_links.has(link) && unique) continue;+ fetched_links.add(link);+ const title = title_tag.textContent || "";+ const description = description_tag.textContent || "";+ fetched_results++;+ new_results++;+ if (link && title && description) {+ start += 1+ results.push({+ url: link,+ title: title,+ description: description+ });+ }+ if (fetched_results >= num_results) break;+ }+ }++ await new Promise((resolve) =>+ setTimeout(resolve, sleep_interval * 1000),+ );+ } catch (error) {+ if (error.message === "Too many requests") {+ logger.warn("Too many requests, breaking the loop");+ break;+ }+ throw error;+ }++ if (start === 0) {+ return results.length > 0 ? { web: results } : {};+ }+ }+ if (attempts >= maxAttempts) {+ logger.warn("Max attempts reached, breaking the loop");+ }+ return results.length > 0 ? { web: results } : {};+}\ No newline at end of filediff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex 38cef0f1f..621a92d46 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -1,6 +1,10 @@import { logger } from "../../lib/logger";-import { SearchResult, SearchV2Response, SearchResultType } from "../../lib/entities";-import { fire_engine_search, fire_engine_search_v2 } from "./fireEngine-v2";+import { SearchV2Response, SearchResultType } from "../../lib/entities";+import { fire_engine_search_v2 } from "./fireEngine-v2";+import { serper_search } from "./serper";+import { searchapi_search } from "./searchapi";+import { searxng_search } from "./searxng";+import { googleSearch } from "./googlesearch";export async function search({query,@@ -44,54 +48,56 @@ export async function search({return results;}+ if (process.env.SERPER_API_KEY) {+ const results = await serper_search(query, {+ num_results,+ tbs,+ filter,+ lang,+ country,+ location,+ });+ if (results.web && results.web.length > 0) return results;+ }+ if (process.env.SEARCHAPI_API_KEY) {+ const results = await searchapi_search(query, {+ num_results,+ tbs,+ filter,+ lang,+ country,+ location,+ });+ if (results.web && results.web.length > 0) return results;+ }+ if (process.env.SEARXNG_ENDPOINT) {+ const results = await searxng_search(query, {+ num_results,+ tbs,+ filter,+ lang,+ country,+ location,+ });+ if (results.web && results.web.length > 0) return results;+ }++ const googleResults = await googleSearch(+ query,+ advanced,+ num_results,+ tbs,+ filter,+ lang,+ country,+ proxy,+ sleep_interval,+ timeout,+ );+ if (googleResults.web && googleResults.web.length > 0) return googleResults;+// Fallback to empty responsereturn {};-- // if (process.env.SERPER_API_KEY) {- // const results = await serper_search(query, {- // num_results,- // tbs,- // filter,- // lang,- // country,- // location,- // });- // if (results.length > 0) return results;- // }- // if (process.env.SEARCHAPI_API_KEY) {- // const results = await searchapi_search(query, {- // num_results,- // tbs,- // filter,- // lang,- // country,- // location,- // });- // if (results.length > 0) return results;- // }- // if (process.env.SEARXNG_ENDPOINT) {- // const results = await searxng_search(query, {- // num_results,- // tbs,- // filter,- // lang,- // country,- // location,- // });- // if (results.length > 0) return results;- // }- // return await googleSearch(- // query,- // advanced,- // num_results,- // tbs,- // filter,- // lang,- // country,- // proxy,- // sleep_interval,- // timeout,- // );} catch (error) {logger.error(`Error in search function`, { error });return {};diff --git a/apps/api/src/search/v2/searchapi.ts b/apps/api/src/search/v2/searchapi.tsnew file mode 100644index 000000000..f1f20af78--- /dev/null+++ b/apps/api/src/search/v2/searchapi.ts@@ -0,0 +1,66 @@+import axios from "axios";+import dotenv from "dotenv";+import { SearchV2Response, WebSearchResult } from "../../lib/entities";++dotenv.config();++interface SearchOptions {+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ num_results: number;+ page?: number;+}++export async function searchapi_search(+ q: string,+ options: SearchOptions,+): Promise<SearchV2Response> {+ const params = {+ q: q,+ hl: options.lang,+ gl: options.country,+ location: options.location,+ num: options.num_results,+ page: options.page ?? 1,+ engine: process.env.SEARCHAPI_ENGINE || "google",+ };++ const url = `https://www.searchapi.io/api/v1/search`;++ try {+ const response = await axios.get(url, {+ headers: {+ Authorization: `Bearer ${process.env.SEARCHAPI_API_KEY}`,+ "Content-Type": "application/json",+ "X-SearchApi-Source": "Firecrawl",+ },+ params: params,+ });++ if (response.status === 401) {+ throw new Error("Unauthorized. Please check your API key.");+ }++ const data = response.data;++ if (data && Array.isArray(data.organic_results)) {+ const webResults: WebSearchResult[] = data.organic_results.map((a: any) => ({+ url: a.link,+ title: a.title,+ description: a.snippet,+ }));++ return {+ web: webResults+ };+ } else {+ return {};+ }+ } catch (error) {+ console.error(`There was an error searching for content: ${error.message}`);+ return {};+ }+}\ No newline at end of filediff --git a/apps/api/src/search/v2/searxng.ts b/apps/api/src/search/v2/searxng.tsnew file mode 100644index 000000000..8761c1497--- /dev/null+++ b/apps/api/src/search/v2/searxng.ts@@ -0,0 +1,68 @@+import axios from "axios";+import dotenv from "dotenv";+import { SearchV2Response, WebSearchResult } from "../../lib/entities";+import { logger } from "../../lib/logger"++dotenv.config();++interface SearchOptions {+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ num_results: number;+ page?: number;+}++export async function searxng_search(+ q: string,+ options: SearchOptions,+): Promise<SearchV2Response> {+ const params = {+ q: q,+ language: options.lang,+ // gl: options.country, //not possible with SearXNG+ // location: options.location, //not possible with SearXNG+ // num: options.num_results, //not possible with SearXNG+ engines: process.env.SEARXNG_ENGINES || "",+ categories: process.env.SEARXNG_CATEGORIES || "",+ pageno: options.page ?? 1,+ format: "json"+ };++ const url = process.env.SEARXNG_ENDPOINT!;+ // Remove trailing slash if it exists+ const cleanedUrl = url.endsWith('/') ? url.slice(0, -1) : url;++ // Concatenate "/search" to the cleaned URL+ const finalUrl = cleanedUrl + "/search";++ try {+ const response = await axios.get(finalUrl, {+ headers: {+ "Content-Type": "application/json",+ },+ params: params,+ });++ const data = response.data;++ if (data && Array.isArray(data.results)) {+ const webResults: WebSearchResult[] = data.results.map((a: any) => ({+ url: a.url,+ title: a.title,+ description: a.content,+ }));++ return {+ web: webResults+ };+ } else {+ return {};+ }+ } catch (error) {+ logger.error(`There was an error searching for content`, { error });+ return {};+ }+}\ No newline at end of filediff --git a/apps/api/src/search/v2/serper.ts b/apps/api/src/search/v2/serper.tsnew file mode 100644index 000000000..7881ed094--- /dev/null+++ b/apps/api/src/search/v2/serper.ts@@ -0,0 +1,57 @@+import axios from "axios";+import dotenv from "dotenv";+import { SearchV2Response, WebSearchResult } from "../../lib/entities";++dotenv.config();++export async function serper_search(+ q,+ options: {+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ num_results: number;+ page?: number;+ },+): Promise<SearchV2Response> {+ let data = JSON.stringify({+ q: q,+ hl: options.lang,+ gl: options.country,+ location: options.location,+ tbs: options.tbs,+ num: options.num_results,+ page: options.page ?? 1,+ });++ let config = {+ method: "POST",+ url: "https://google.serper.dev/search",+ headers: {+ "X-API-KEY": process.env.SERPER_API_KEY,+ "Content-Type": "application/json",+ },+ data: data,+ };++ try {+ const response = await axios(config);+ if (response && response.data && Array.isArray(response.data.organic)) {+ const webResults: WebSearchResult[] = response.data.organic.map((a) => ({+ url: a.link,+ title: a.title,+ description: a.snippet,+ }));++ return {+ web: webResults+ };+ } else {+ return {};+ }+ } catch (error) {+ return {};+ }+}\ No newline at end of file