Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/search): improved logging
Gergő Móricz3f9e89367ea6
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 5ad0f4dff..6690f1f9b 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -60,11 +60,12 @@ export async function searchHelper(const num_results_buffer = Math.floor(num_results * 1.5);let res = await search({- query: query,- advanced: advanced,+ query,+ logger,+ advanced,num_results: num_results_buffer,- tbs: tbs,- filter: filter,+ tbs,+ filter,lang: searchOptions.lang ?? "en",country: searchOptions.country ?? "us",location: searchOptions.location,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex f185ce4df..6f5a0b7ea 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -48,6 +48,7 @@ export async function searchAndScrapeSearchResult(try {const searchResults = await search({query,+ logger,num_results: 5,});@@ -242,6 +243,7 @@ export async function searchController(let searchResults = await search({query: req.body.query,+ logger,advanced: false,num_results: num_results_buffer,tbs: req.body.tbs,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex ff0a66659..980ce770c 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -45,6 +45,7 @@ export async function searchAndScrapeX402SearchResult(try {const searchResults = await search({query,+ logger,num_results: 5,});@@ -238,6 +239,7 @@ export async function x402SearchController(let searchResults = await search({query: req.body.query,+ logger,advanced: false,num_results: num_results_buffer,tbs: req.body.tbs,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex f3815db91..8269d08d9 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -206,6 +206,7 @@ export async function searchController(const searchResponse = await search({query: searchQuery,+ logger,advanced: false,num_results: num_results_buffer,tbs: req.body.tbs,diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 65215c60b..4d5f2bf20 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -176,6 +176,7 @@ export async function performExtraction(let rptxt = rephrasedPrompt?.text.replace('"', "").replace("'", "") || "";const searchResults = await search({query: rptxt,+ logger,num_results: 10,});diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 079489238..212af66c2 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -108,6 +108,7 @@ import { langfuse } from "../../../services/langfuse";const rephrasedPrompt = await generateBasicCompletion_FO(buildRephraseToSerpPrompt_F0(request.prompt), { teamId, extractId });const searchResults = await search({query: rephrasedPrompt.replace('"', "").replace("'", ""),+ logger,num_results: 10,});diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.tsindex f0a2d6125..666a51a79 100644--- a/apps/api/src/search/index.ts+++ b/apps/api/src/search/index.ts@@ -1,13 +1,14 @@-import { logger } from "../../src/lib/logger";import { SearchResult } from "../../src/lib/entities";import { googleSearch } from "./googlesearch";import { searchapi_search } from "./searchapi";import { serper_search } from "./serper";import { searxng_search } from "./searxng";import { fire_engine_search } from "./fireEngine";+import { Logger } from "winston";export async function search({query,+ logger,advanced = false,num_results = 5,tbs = undefined,@@ -20,6 +21,7 @@ export async function search({timeout = 5000,}: {query: string;+ logger: Logger;advanced?: boolean;num_results?: number;tbs?: string;@@ -33,6 +35,7 @@ export async function search({}): Promise<SearchResult[]> {try {if (process.env.FIRE_ENGINE_BETA_URL) {+ logger.info("Using fire engine search");const results = await fire_engine_search(query, {numResults: num_results,tbs,@@ -44,6 +47,7 @@ export async function search({if (results.length > 0) return results;}if (process.env.SERPER_API_KEY) {+ logger.info("Using serper search");const results = await serper_search(query, {num_results,tbs,@@ -55,6 +59,7 @@ export async function search({if (results.length > 0) return results;}if (process.env.SEARCHAPI_API_KEY) {+ logger.info("Using searchapi search");const results = await searchapi_search(query, {num_results,tbs,@@ -66,6 +71,7 @@ export async function search({if (results.length > 0) return results;}if (process.env.SEARXNG_ENDPOINT) {+ logger.info("Using searxng search");const results = await searxng_search(query, {num_results,tbs,@@ -76,6 +82,7 @@ export async function search({});if (results.length > 0) return results;}+ logger.info("Using google search");return await googleSearch(query,advanced,diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex 621a92d46..6557b50d0 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -5,9 +5,11 @@ import { serper_search } from "./serper";import { searchapi_search } from "./searchapi";import { searxng_search } from "./searxng";import { googleSearch } from "./googlesearch";+import { Logger } from "winston";export async function search({query,+ logger,advanced = false,num_results = 5,tbs = undefined,@@ -21,6 +23,7 @@ export async function search({type = undefined,}: {query: string;+ logger: Logger;advanced?: boolean;num_results?: number;tbs?: string;@@ -35,6 +38,7 @@ export async function search({}): Promise<SearchV2Response> {try {if (process.env.FIRE_ENGINE_BETA_URL) {+ logger.info("Using fire engine search");const results = await fire_engine_search_v2(query, {numResults: num_results,tbs,@@ -49,6 +53,7 @@ export async function search({}if (process.env.SERPER_API_KEY) {+ logger.info("Using serper search");const results = await serper_search(query, {num_results,tbs,@@ -60,6 +65,7 @@ export async function search({if (results.web && results.web.length > 0) return results;}if (process.env.SEARCHAPI_API_KEY) {+ logger.info("Using searchapi search");const results = await searchapi_search(query, {num_results,tbs,@@ -71,6 +77,7 @@ export async function search({if (results.web && results.web.length > 0) return results;}if (process.env.SEARXNG_ENDPOINT) {+ logger.info("Using searxng search");const results = await searxng_search(query, {num_results,tbs,@@ -82,6 +89,7 @@ export async function search({if (results.web && results.web.length > 0) return results;}+ logger.info("Using google search");const googleResults = await googleSearch(query,advanced,