Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: rm external search apis (#2420)
Nicolasd05fc8cb0e55
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 11fa322f5..bce12dbc7 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -257,7 +257,6 @@ jobs:# FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}# FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}# USE_DB_AUTHENTICATION: true- # SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}# ENV: ${{ secrets.ENV }}# RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }}# RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }}diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.tsindex 2948e8f58..40fd00d70 100644--- a/apps/api/src/search/index.ts+++ b/apps/api/src/search/index.ts@@ -1,6 +1,4 @@import { SearchResult } from "../../src/lib/entities";-import { searchapi_search } from "./searchapi";-import { serper_search } from "./serper";import { searxng_search } from "./searxng";import { fire_engine_search } from "./fireEngine";import { Logger } from "winston";@@ -46,30 +44,6 @@ export async function search({});return results;}- if (process.env.SERPER_API_KEY) {- logger.info("Using serper search");- const results = await serper_search(query, {- num_results,- tbs,- filter,- lang,- country,- location,- });- if (results.length > 0) return results;- }- if (process.env.SEARCHAPI_API_KEY) {- logger.info("Using searchapi search");- const results = await searchapi_search(query, {- num_results,- tbs,- filter,- lang,- country,- location,- });- if (results.length > 0) return results;- }if (process.env.SEARXNG_ENDPOINT) {logger.info("Using searxng search");const results = await searxng_search(query, {diff --git a/apps/api/src/search/searchapi.ts b/apps/api/src/search/searchapi.tsdeleted file mode 100644index 896c64c69..000000000--- a/apps/api/src/search/searchapi.ts+++ /dev/null@@ -1,62 +0,0 @@-import axios from "axios";-import dotenv from "dotenv";-import { SearchResult } from "../../src/lib/entities";--dotenv.config();--interface SearchOptions {- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- num_results: number;- page?: number;-}--export async function searchapi_search(- q: string,- options: SearchOptions,-): Promise<SearchResult[]> {- const params = {- q: q,- hl: options.lang,- gl: options.country,- location: options.location,- num: options.num_results,- page: options.page ?? 1,- engine: process.env.SEARCHAPI_ENGINE || "google",- };-- const url = `https://www.searchapi.io/api/v1/search`;-- try {- const response = await axios.get(url, {- headers: {- Authorization: `Bearer ${process.env.SEARCHAPI_API_KEY}`,- "Content-Type": "application/json",- "X-SearchApi-Source": "Firecrawl",- },- params: params,- });-- if (response.status === 401) {- throw new Error("Unauthorized. Please check your API key.");- }-- const data = response.data;-- if (data && Array.isArray(data.organic_results)) {- return data.organic_results.map((a: any) => ({- url: a.link,- title: a.title,- description: a.snippet,- }));- } else {- return [];- }- } catch (error) {- console.error(`There was an error searching for content: ${error.message}`);- return [];- }-}diff --git a/apps/api/src/search/serper.ts b/apps/api/src/search/serper.tsdeleted file mode 100644index 901b0b67b..000000000--- a/apps/api/src/search/serper.ts+++ /dev/null@@ -1,48 +0,0 @@-import axios from "axios";-import dotenv from "dotenv";-import { SearchResult } from "../../src/lib/entities";--dotenv.config();--export async function serper_search(- q,- options: {- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- num_results: number;- page?: number;- },-): Promise<SearchResult[]> {- let data = JSON.stringify({- q: q,- hl: options.lang,- gl: options.country,- location: options.location,- tbs: options.tbs,- num: options.num_results,- page: options.page ?? 1,- });-- let config = {- method: "POST",- url: "https://google.serper.dev/search",- headers: {- "X-API-KEY": process.env.SERPER_API_KEY,- "Content-Type": "application/json",- },- data: data,- };- const response = await axios(config);- if (response && response.data && Array.isArray(response.data.organic)) {- return response.data.organic.map(a => ({- url: a.link,- title: a.title,- description: a.snippet,- }));- } else {- return [];- }-}diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex 13bb73f47..5125c8c77 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -1,7 +1,5 @@import { SearchV2Response, SearchResultType } from "../../lib/entities";import { fire_engine_search_v2 } from "./fireEngine-v2";-import { serper_search } from "./serper";-import { searchapi_search } from "./searchapi";import { searxng_search } from "./searxng";import { ddgSearch } from "./ddgsearch";import { Logger } from "winston";@@ -51,30 +49,6 @@ export async function search({return results;}- if (process.env.SERPER_API_KEY) {- logger.info("Using serper search");- const results = await serper_search(query, {- num_results,- tbs,- filter,- lang,- country,- location,- });- if (results.web && results.web.length > 0) return results;- }- if (process.env.SEARCHAPI_API_KEY) {- logger.info("Using searchapi search");- const results = await searchapi_search(query, {- num_results,- tbs,- filter,- lang,- country,- location,- });- if (results.web && results.web.length > 0) return results;- }if (process.env.SEARXNG_ENDPOINT) {logger.info("Using searxng search");const results = await searxng_search(query, {diff --git a/apps/api/src/search/v2/searchapi.ts b/apps/api/src/search/v2/searchapi.tsdeleted file mode 100644index 77069db34..000000000--- a/apps/api/src/search/v2/searchapi.ts+++ /dev/null@@ -1,68 +0,0 @@-import axios from "axios";-import dotenv from "dotenv";-import { SearchV2Response, WebSearchResult } from "../../lib/entities";--dotenv.config();--interface SearchOptions {- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- num_results: number;- page?: number;-}--export async function searchapi_search(- q: string,- options: SearchOptions,-): Promise<SearchV2Response> {- const params = {- q: q,- hl: options.lang,- gl: options.country,- location: options.location,- num: options.num_results,- page: options.page ?? 1,- engine: process.env.SEARCHAPI_ENGINE || "google",- };-- const url = `https://www.searchapi.io/api/v1/search`;-- try {- const response = await axios.get(url, {- headers: {- Authorization: `Bearer ${process.env.SEARCHAPI_API_KEY}`,- "Content-Type": "application/json",- "X-SearchApi-Source": "Firecrawl",- },- params: params,- });-- if (response.status === 401) {- throw new Error("Unauthorized. Please check your API key.");- }-- const data = response.data;-- if (data && Array.isArray(data.organic_results)) {- const webResults: WebSearchResult[] = data.organic_results.map(- (a: any) => ({- url: a.link,- title: a.title,- description: a.snippet,- }),- );-- return {- web: webResults,- };- } else {- return {};- }- } catch (error) {- console.error(`There was an error searching for content: ${error.message}`);- return {};- }-}diff --git a/apps/api/src/search/v2/serper.ts b/apps/api/src/search/v2/serper.tsdeleted file mode 100644index 95c758350..000000000--- a/apps/api/src/search/v2/serper.ts+++ /dev/null@@ -1,57 +0,0 @@-import axios from "axios";-import dotenv from "dotenv";-import { SearchV2Response, WebSearchResult } from "../../lib/entities";--dotenv.config();--export async function serper_search(- q,- options: {- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- num_results: number;- page?: number;- },-): Promise<SearchV2Response> {- let data = JSON.stringify({- q: q,- hl: options.lang,- gl: options.country,- location: options.location,- tbs: options.tbs,- num: options.num_results,- page: options.page ?? 1,- });-- let config = {- method: "POST",- url: "https://google.serper.dev/search",- headers: {- "X-API-KEY": process.env.SERPER_API_KEY,- "Content-Type": "application/json",- },- data: data,- };-- try {- const response = await axios(config);- if (response && response.data && Array.isArray(response.data.organic)) {- const webResults: WebSearchResult[] = response.data.organic.map(a => ({- url: a.link,- title: a.title,- description: a.snippet,- }));-- return {- web: webResults,- };- } else {- return {};- }- } catch (error) {- return {};- }-}diff --git a/docker-compose.yaml b/docker-compose.yamlindex 5123cd1c2..c71676785 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -35,8 +35,6 @@ x-common-env: &common-envSUPABASE_URL: ${SUPABASE_URL}SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}- SERPER_API_KEY: ${SERPER_API_KEY}- SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY}LOGGING_LEVEL: ${LOGGING_LEVEL}PROXY_SERVER: ${PROXY_SERVER}PROXY_USERNAME: ${PROXY_USERNAME}