Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: rm external search apis (#2420)

Nicolasd05fc8cb0e55
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 11fa322f5..bce12dbc7 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -257,7 +257,6 @@ jobs:
# FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}
# FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}
# USE_DB_AUTHENTICATION: true
- # SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}
# ENV: ${{ secrets.ENV }}
# RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}
# RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}
diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts
index 2948e8f58..40fd00d70 100644
--- a/apps/api/src/search/index.ts
+++ b/apps/api/src/search/index.ts
@@ -1,6 +1,4 @@
import { SearchResult } from "../../src/lib/entities";
-import { searchapi_search } from "./searchapi";
-import { serper_search } from "./serper";
import { searxng_search } from "./searxng";
import { fire_engine_search } from "./fireEngine";
import { Logger } from "winston";
@@ -46,30 +44,6 @@ export async function search({
});
return results;
}
- if (process.env.SERPER_API_KEY) {
- logger.info("Using serper search");
- const results = await serper_search(query, {
- num_results,
- tbs,
- filter,
- lang,
- country,
- location,
- });
- if (results.length > 0) return results;
- }
- if (process.env.SEARCHAPI_API_KEY) {
- logger.info("Using searchapi search");
- const results = await searchapi_search(query, {
- num_results,
- tbs,
- filter,
- lang,
- country,
- location,
- });
- if (results.length > 0) return results;
- }
if (process.env.SEARXNG_ENDPOINT) {
logger.info("Using searxng search");
const results = await searxng_search(query, {
diff --git a/apps/api/src/search/searchapi.ts b/apps/api/src/search/searchapi.ts
deleted file mode 100644
index 896c64c69..000000000
--- a/apps/api/src/search/searchapi.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-import axios from "axios";
-import dotenv from "dotenv";
-import { SearchResult } from "../../src/lib/entities";
-
-dotenv.config();
-
-interface SearchOptions {
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- num_results: number;
- page?: number;
-}
-
-export async function searchapi_search(
- q: string,
- options: SearchOptions,
-): Promise<SearchResult[]> {
- const params = {
- q: q,
- hl: options.lang,
- gl: options.country,
- location: options.location,
- num: options.num_results,
- page: options.page ?? 1,
- engine: process.env.SEARCHAPI_ENGINE || "google",
- };
-
- const url = `https://www.searchapi.io/api/v1/search`;
-
- try {
- const response = await axios.get(url, {
- headers: {
- Authorization: `Bearer ${process.env.SEARCHAPI_API_KEY}`,
- "Content-Type": "application/json",
- "X-SearchApi-Source": "Firecrawl",
- },
- params: params,
- });
-
- if (response.status === 401) {
- throw new Error("Unauthorized. Please check your API key.");
- }
-
- const data = response.data;
-
- if (data && Array.isArray(data.organic_results)) {
- return data.organic_results.map((a: any) => ({
- url: a.link,
- title: a.title,
- description: a.snippet,
- }));
- } else {
- return [];
- }
- } catch (error) {
- console.error(`There was an error searching for content: ${error.message}`);
- return [];
- }
-}
diff --git a/apps/api/src/search/serper.ts b/apps/api/src/search/serper.ts
deleted file mode 100644
index 901b0b67b..000000000
--- a/apps/api/src/search/serper.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-import axios from "axios";
-import dotenv from "dotenv";
-import { SearchResult } from "../../src/lib/entities";
-
-dotenv.config();
-
-export async function serper_search(
- q,
- options: {
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- num_results: number;
- page?: number;
- },
-): Promise<SearchResult[]> {
- let data = JSON.stringify({
- q: q,
- hl: options.lang,
- gl: options.country,
- location: options.location,
- tbs: options.tbs,
- num: options.num_results,
- page: options.page ?? 1,
- });
-
- let config = {
- method: "POST",
- url: "https://google.serper.dev/search",
- headers: {
- "X-API-KEY": process.env.SERPER_API_KEY,
- "Content-Type": "application/json",
- },
- data: data,
- };
- const response = await axios(config);
- if (response && response.data && Array.isArray(response.data.organic)) {
- return response.data.organic.map(a => ({
- url: a.link,
- title: a.title,
- description: a.snippet,
- }));
- } else {
- return [];
- }
-}
diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.ts
index 13bb73f47..5125c8c77 100644
--- a/apps/api/src/search/v2/index.ts
+++ b/apps/api/src/search/v2/index.ts
@@ -1,7 +1,5 @@
import { SearchV2Response, SearchResultType } from "../../lib/entities";
import { fire_engine_search_v2 } from "./fireEngine-v2";
-import { serper_search } from "./serper";
-import { searchapi_search } from "./searchapi";
import { searxng_search } from "./searxng";
import { ddgSearch } from "./ddgsearch";
import { Logger } from "winston";
@@ -51,30 +49,6 @@ export async function search({
return results;
}
- if (process.env.SERPER_API_KEY) {
- logger.info("Using serper search");
- const results = await serper_search(query, {
- num_results,
- tbs,
- filter,
- lang,
- country,
- location,
- });
- if (results.web && results.web.length > 0) return results;
- }
- if (process.env.SEARCHAPI_API_KEY) {
- logger.info("Using searchapi search");
- const results = await searchapi_search(query, {
- num_results,
- tbs,
- filter,
- lang,
- country,
- location,
- });
- if (results.web && results.web.length > 0) return results;
- }
if (process.env.SEARXNG_ENDPOINT) {
logger.info("Using searxng search");
const results = await searxng_search(query, {
diff --git a/apps/api/src/search/v2/searchapi.ts b/apps/api/src/search/v2/searchapi.ts
deleted file mode 100644
index 77069db34..000000000
--- a/apps/api/src/search/v2/searchapi.ts
+++ /dev/null
@@ -1,68 +0,0 @@
-import axios from "axios";
-import dotenv from "dotenv";
-import { SearchV2Response, WebSearchResult } from "../../lib/entities";
-
-dotenv.config();
-
-interface SearchOptions {
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- num_results: number;
- page?: number;
-}
-
-export async function searchapi_search(
- q: string,
- options: SearchOptions,
-): Promise<SearchV2Response> {
- const params = {
- q: q,
- hl: options.lang,
- gl: options.country,
- location: options.location,
- num: options.num_results,
- page: options.page ?? 1,
- engine: process.env.SEARCHAPI_ENGINE || "google",
- };
-
- const url = `https://www.searchapi.io/api/v1/search`;
-
- try {
- const response = await axios.get(url, {
- headers: {
- Authorization: `Bearer ${process.env.SEARCHAPI_API_KEY}`,
- "Content-Type": "application/json",
- "X-SearchApi-Source": "Firecrawl",
- },
- params: params,
- });
-
- if (response.status === 401) {
- throw new Error("Unauthorized. Please check your API key.");
- }
-
- const data = response.data;
-
- if (data && Array.isArray(data.organic_results)) {
- const webResults: WebSearchResult[] = data.organic_results.map(
- (a: any) => ({
- url: a.link,
- title: a.title,
- description: a.snippet,
- }),
- );
-
- return {
- web: webResults,
- };
- } else {
- return {};
- }
- } catch (error) {
- console.error(`There was an error searching for content: ${error.message}`);
- return {};
- }
-}
diff --git a/apps/api/src/search/v2/serper.ts b/apps/api/src/search/v2/serper.ts
deleted file mode 100644
index 95c758350..000000000
--- a/apps/api/src/search/v2/serper.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-import axios from "axios";
-import dotenv from "dotenv";
-import { SearchV2Response, WebSearchResult } from "../../lib/entities";
-
-dotenv.config();
-
-export async function serper_search(
- q,
- options: {
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- num_results: number;
- page?: number;
- },
-): Promise<SearchV2Response> {
- let data = JSON.stringify({
- q: q,
- hl: options.lang,
- gl: options.country,
- location: options.location,
- tbs: options.tbs,
- num: options.num_results,
- page: options.page ?? 1,
- });
-
- let config = {
- method: "POST",
- url: "https://google.serper.dev/search",
- headers: {
- "X-API-KEY": process.env.SERPER_API_KEY,
- "Content-Type": "application/json",
- },
- data: data,
- };
-
- try {
- const response = await axios(config);
- if (response && response.data && Array.isArray(response.data.organic)) {
- const webResults: WebSearchResult[] = response.data.organic.map(a => ({
- url: a.link,
- title: a.title,
- description: a.snippet,
- }));
-
- return {
- web: webResults,
- };
- } else {
- return {};
- }
- } catch (error) {
- return {};
- }
-}
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 5123cd1c2..c71676785 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -35,8 +35,6 @@ x-common-env: &common-env
SUPABASE_URL: ${SUPABASE_URL}
SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
- SERPER_API_KEY: ${SERPER_API_KEY}
- SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY}
LOGGING_LEVEL: ${LOGGING_LEVEL}
PROXY_SERVER: ${PROXY_SERVER}
PROXY_USERNAME: ${PROXY_USERNAME}