Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix search

Gergő Móriczd3bda3f011a6
diff --git a/apps/api/src/search/v2/googlesearch.ts b/apps/api/src/search/v2/googlesearch.ts
new file mode 100644
index 000000000..f705a5430
--- /dev/null
+++ b/apps/api/src/search/v2/googlesearch.ts
@@ -0,0 +1,169 @@
+import axios from "axios";
+import { JSDOM } from 'jsdom';
+import { SearchV2Response, WebSearchResult } from "../../lib/entities";
+import { logger } from "../../lib/logger";
+import https from 'https';
+
+const getRandomInt = (min: number, max: number): number => Math.floor(Math.random() * (max - min + 1)) + min;
+
+export function get_useragent(): string {
+ const lynx_version = `Lynx/${getRandomInt(2, 3)}.${getRandomInt(8, 9)}.${getRandomInt(0, 2)}`;
+ const libwww_version = `libwww-FM/${getRandomInt(2, 3)}.${getRandomInt(13, 15)}`;
+ const ssl_mm_version = `SSL-MM/${getRandomInt(1, 2)}.${getRandomInt(3, 5)}`;
+ const openssl_version = `OpenSSL/${getRandomInt(1, 3)}.${getRandomInt(0, 4)}.${getRandomInt(0, 9)}`;
+ return `${lynx_version} ${libwww_version} ${ssl_mm_version} ${openssl_version}`;
+}
+
+async function _req(
+ term: string,
+ results: number,
+ lang: string,
+ country: string,
+ start: number,
+ proxies: any,
+ timeout: number,
+ tbs: string | undefined = undefined,
+ filter: string | undefined = undefined,
+) {
+ const params = {
+ q: term,
+ num: results+2, // Number of results to return
+ hl: lang,
+ gl: country,
+ safe: "active",
+ start: start,
+ };
+ if (tbs) {
+ params["tbs"] = tbs;
+ }
+ if (filter) {
+ params["filter"] = filter;
+ }
+ var agent = get_useragent();
+ try {
+ const resp = await axios.get("https://www.google.com/search", {
+ headers: {
+ "User-Agent": agent,
+ "Accept": "*/*"
+ },
+ params: params,
+ proxy: proxies,
+ timeout: timeout,
+ httpsAgent: new https.Agent({
+ rejectUnauthorized: true
+ }),
+ withCredentials: true
+ });
+ return resp;
+ } catch (error) {
+ if (error.response && error.response.status === 429) {
+ logger.warn("Google Search: Too many requests, try again later.", {
+ status: error.response.status,
+ statusText: error.response.statusText
+ });
+ throw new Error("Google Search: Too many requests, try again later.");
+ }
+ throw error;
+ }
+}
+
+export async function googleSearch(
+ term: string,
+ advanced = false,
+ num_results = 5,
+ tbs = undefined as string | undefined,
+ filter = undefined as string | undefined,
+ lang = "en",
+ country = "us",
+ proxy = undefined as string | undefined,
+ sleep_interval = 0,
+ timeout = 5000,
+): Promise<SearchV2Response> {
+ let proxies: any = null;
+ if (proxy) {
+ if (proxy.startsWith("https")) {
+ proxies = { https: proxy };
+ } else {
+ proxies = { http: proxy };
+ }
+ }
+
+ // TODO: knowledge graph, answer box, etc.
+
+ let start = 0;
+ let results: WebSearchResult[] = [];
+ let attempts = 0;
+ const maxAttempts = 20; // Define a maximum number of attempts to prevent infinite loop
+ while (start < num_results && attempts < maxAttempts) {
+ try {
+ const resp = await _req(
+ term,
+ num_results - start,
+ lang,
+ country,
+ start,
+ proxies,
+ timeout,
+ tbs,
+ filter,
+ );
+ const dom = new JSDOM(resp.data);
+ const document = dom.window.document;
+ const result_block = document.querySelectorAll("div.ezO2md");
+ let new_results = 0;
+ let unique = true;
+ let fetched_results = 0;
+
+ const fetched_links = new Set<string>();
+ if (result_block.length === 0) {
+ start += 1;
+ attempts += 1;
+ } else {
+ attempts = 0;
+ }
+
+ for (const result of result_block) {
+ const link_tag = result.querySelector("a[href]") as HTMLAnchorElement;
+ const title_tag = link_tag ? link_tag.querySelector("span.CVA68e") : null;
+ const description_tag = result.querySelector("span.FrIlee");
+
+ if (link_tag && title_tag && description_tag) {
+ const link = decodeURIComponent(link_tag.href.split("&")[0].replace("/url?q=", ""));
+ if (fetched_links.has(link) && unique) continue;
+ fetched_links.add(link);
+ const title = title_tag.textContent || "";
+ const description = description_tag.textContent || "";
+ fetched_results++;
+ new_results++;
+ if (link && title && description) {
+ start += 1
+ results.push({
+ url: link,
+ title: title,
+ description: description
+ });
+ }
+ if (fetched_results >= num_results) break;
+ }
+ }
+
+ await new Promise((resolve) =>
+ setTimeout(resolve, sleep_interval * 1000),
+ );
+ } catch (error) {
+ if (error.message === "Too many requests") {
+ logger.warn("Too many requests, breaking the loop");
+ break;
+ }
+ throw error;
+ }
+
+ if (start === 0) {
+ return results.length > 0 ? { web: results } : {};
+ }
+ }
+ if (attempts >= maxAttempts) {
+ logger.warn("Max attempts reached, breaking the loop");
+ }
+ return results.length > 0 ? { web: results } : {};
+}
\ No newline at end of file
diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.ts
index 38cef0f1f..621a92d46 100644
--- a/apps/api/src/search/v2/index.ts
+++ b/apps/api/src/search/v2/index.ts
@@ -1,6 +1,10 @@
import { logger } from "../../lib/logger";
-import { SearchResult, SearchV2Response, SearchResultType } from "../../lib/entities";
-import { fire_engine_search, fire_engine_search_v2 } from "./fireEngine-v2";
+import { SearchV2Response, SearchResultType } from "../../lib/entities";
+import { fire_engine_search_v2 } from "./fireEngine-v2";
+import { serper_search } from "./serper";
+import { searchapi_search } from "./searchapi";
+import { searxng_search } from "./searxng";
+import { googleSearch } from "./googlesearch";
export async function search({
query,
@@ -44,54 +48,56 @@ export async function search({
return results;
}
+ if (process.env.SERPER_API_KEY) {
+ const results = await serper_search(query, {
+ num_results,
+ tbs,
+ filter,
+ lang,
+ country,
+ location,
+ });
+ if (results.web && results.web.length > 0) return results;
+ }
+ if (process.env.SEARCHAPI_API_KEY) {
+ const results = await searchapi_search(query, {
+ num_results,
+ tbs,
+ filter,
+ lang,
+ country,
+ location,
+ });
+ if (results.web && results.web.length > 0) return results;
+ }
+ if (process.env.SEARXNG_ENDPOINT) {
+ const results = await searxng_search(query, {
+ num_results,
+ tbs,
+ filter,
+ lang,
+ country,
+ location,
+ });
+ if (results.web && results.web.length > 0) return results;
+ }
+
+ const googleResults = await googleSearch(
+ query,
+ advanced,
+ num_results,
+ tbs,
+ filter,
+ lang,
+ country,
+ proxy,
+ sleep_interval,
+ timeout,
+ );
+ if (googleResults.web && googleResults.web.length > 0) return googleResults;
+
// Fallback to empty response
return {};
-
- // if (process.env.SERPER_API_KEY) {
- // const results = await serper_search(query, {
- // num_results,
- // tbs,
- // filter,
- // lang,
- // country,
- // location,
- // });
- // if (results.length > 0) return results;
- // }
- // if (process.env.SEARCHAPI_API_KEY) {
- // const results = await searchapi_search(query, {
- // num_results,
- // tbs,
- // filter,
- // lang,
- // country,
- // location,
- // });
- // if (results.length > 0) return results;
- // }
- // if (process.env.SEARXNG_ENDPOINT) {
- // const results = await searxng_search(query, {
- // num_results,
- // tbs,
- // filter,
- // lang,
- // country,
- // location,
- // });
- // if (results.length > 0) return results;
- // }
- // return await googleSearch(
- // query,
- // advanced,
- // num_results,
- // tbs,
- // filter,
- // lang,
- // country,
- // proxy,
- // sleep_interval,
- // timeout,
- // );
} catch (error) {
logger.error(`Error in search function`, { error });
return {};
diff --git a/apps/api/src/search/v2/searchapi.ts b/apps/api/src/search/v2/searchapi.ts
new file mode 100644
index 000000000..f1f20af78
--- /dev/null
+++ b/apps/api/src/search/v2/searchapi.ts
@@ -0,0 +1,66 @@
+import axios from "axios";
+import dotenv from "dotenv";
+import { SearchV2Response, WebSearchResult } from "../../lib/entities";
+
+dotenv.config();
+
+interface SearchOptions {
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ num_results: number;
+ page?: number;
+}
+
+export async function searchapi_search(
+ q: string,
+ options: SearchOptions,
+): Promise<SearchV2Response> {
+ const params = {
+ q: q,
+ hl: options.lang,
+ gl: options.country,
+ location: options.location,
+ num: options.num_results,
+ page: options.page ?? 1,
+ engine: process.env.SEARCHAPI_ENGINE || "google",
+ };
+
+ const url = `https://www.searchapi.io/api/v1/search`;
+
+ try {
+ const response = await axios.get(url, {
+ headers: {
+ Authorization: `Bearer ${process.env.SEARCHAPI_API_KEY}`,
+ "Content-Type": "application/json",
+ "X-SearchApi-Source": "Firecrawl",
+ },
+ params: params,
+ });
+
+ if (response.status === 401) {
+ throw new Error("Unauthorized. Please check your API key.");
+ }
+
+ const data = response.data;
+
+ if (data && Array.isArray(data.organic_results)) {
+ const webResults: WebSearchResult[] = data.organic_results.map((a: any) => ({
+ url: a.link,
+ title: a.title,
+ description: a.snippet,
+ }));
+
+ return {
+ web: webResults
+ };
+ } else {
+ return {};
+ }
+ } catch (error) {
+ console.error(`There was an error searching for content: ${error.message}`);
+ return {};
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/search/v2/searxng.ts b/apps/api/src/search/v2/searxng.ts
new file mode 100644
index 000000000..8761c1497
--- /dev/null
+++ b/apps/api/src/search/v2/searxng.ts
@@ -0,0 +1,68 @@
+import axios from "axios";
+import dotenv from "dotenv";
+import { SearchV2Response, WebSearchResult } from "../../lib/entities";
+import { logger } from "../../lib/logger"
+
+dotenv.config();
+
+interface SearchOptions {
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ num_results: number;
+ page?: number;
+}
+
+export async function searxng_search(
+ q: string,
+ options: SearchOptions,
+): Promise<SearchV2Response> {
+ const params = {
+ q: q,
+ language: options.lang,
+ // gl: options.country, //not possible with SearXNG
+ // location: options.location, //not possible with SearXNG
+ // num: options.num_results, //not possible with SearXNG
+ engines: process.env.SEARXNG_ENGINES || "",
+ categories: process.env.SEARXNG_CATEGORIES || "",
+ pageno: options.page ?? 1,
+ format: "json"
+ };
+
+ const url = process.env.SEARXNG_ENDPOINT!;
+ // Remove trailing slash if it exists
+ const cleanedUrl = url.endsWith('/') ? url.slice(0, -1) : url;
+
+ // Concatenate "/search" to the cleaned URL
+ const finalUrl = cleanedUrl + "/search";
+
+ try {
+ const response = await axios.get(finalUrl, {
+ headers: {
+ "Content-Type": "application/json",
+ },
+ params: params,
+ });
+
+ const data = response.data;
+
+ if (data && Array.isArray(data.results)) {
+ const webResults: WebSearchResult[] = data.results.map((a: any) => ({
+ url: a.url,
+ title: a.title,
+ description: a.content,
+ }));
+
+ return {
+ web: webResults
+ };
+ } else {
+ return {};
+ }
+ } catch (error) {
+ logger.error(`There was an error searching for content`, { error });
+ return {};
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/search/v2/serper.ts b/apps/api/src/search/v2/serper.ts
new file mode 100644
index 000000000..7881ed094
--- /dev/null
+++ b/apps/api/src/search/v2/serper.ts
@@ -0,0 +1,57 @@
+import axios from "axios";
+import dotenv from "dotenv";
+import { SearchV2Response, WebSearchResult } from "../../lib/entities";
+
+dotenv.config();
+
+export async function serper_search(
+ q,
+ options: {
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ num_results: number;
+ page?: number;
+ },
+): Promise<SearchV2Response> {
+ let data = JSON.stringify({
+ q: q,
+ hl: options.lang,
+ gl: options.country,
+ location: options.location,
+ tbs: options.tbs,
+ num: options.num_results,
+ page: options.page ?? 1,
+ });
+
+ let config = {
+ method: "POST",
+ url: "https://google.serper.dev/search",
+ headers: {
+ "X-API-KEY": process.env.SERPER_API_KEY,
+ "Content-Type": "application/json",
+ },
+ data: data,
+ };
+
+ try {
+ const response = await axios(config);
+ if (response && response.data && Array.isArray(response.data.organic)) {
+ const webResults: WebSearchResult[] = response.data.organic.map((a) => ({
+ url: a.link,
+ title: a.title,
+ description: a.snippet,
+ }));
+
+ return {
+ web: webResults
+ };
+ } else {
+ return {};
+ }
+ } catch (error) {
+ return {};
+ }
+}
\ No newline at end of file