Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Implement num_results for searxng (#2645)
Abimael Martell954cd969a413
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.tsindex e2f089f03..cbdfa67f4 100644--- a/apps/api/src/__tests__/snips/v2/search.test.ts+++ b/apps/api/src/__tests__/snips/v2/search.test.ts@@ -6,6 +6,7 @@ import {TEST_PRODUCTION,} from "../lib";import { search, idmux, Identity } from "./lib";+import { config } from "../../../config";let identity: Identity;@@ -214,4 +215,39 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("Search tests", () => {},60000,);++ // SEARXNG-specific pagination tests+ concurrentIf(!!config.SEARXNG_ENDPOINT)(+ "searxng respects limit of 2 results",+ async () => {+ const res = await search(+ {+ query: "firecrawl",+ limit: 2,+ },+ identity,+ );+ expect(res.web).toBeDefined();+ expect(res.web?.length).toBeGreaterThan(0);+ expect(res.web?.length).toBeLessThanOrEqual(2);+ },+ 60000,+ );++ concurrentIf(!!config.SEARXNG_ENDPOINT)(+ "searxng fetches multiple pages for 21 results",+ async () => {+ const res = await search(+ {+ query: "firecrawl",+ limit: 21,+ },+ identity,+ );+ expect(res.web).toBeDefined();+ expect(res.web?.length).toBeGreaterThan(0);+ expect(res.web?.length).toBeLessThanOrEqual(21);+ },+ 60000,+ );});diff --git a/apps/api/src/search/searxng.ts b/apps/api/src/search/searxng.tsindex 46017dfa0..74ab0f10a 100644--- a/apps/api/src/search/searxng.ts+++ b/apps/api/src/search/searxng.ts@@ -17,26 +17,27 @@ export async function searxng_search(q: string,options: SearchOptions,): Promise<SearchResult[]> {- const params = {- q: q,- language: options.lang,- // gl: options.country, //not possible with SearXNG- // location: options.location, //not possible with SearXNG- // num: options.num_results, //not possible with SearXNG- engines: config.SEARXNG_ENGINES ?? "",- categories: config.SEARXNG_CATEGORIES ?? "",- pageno: options.page ?? 1,- format: "json",- };+ const resultsPerPage = 20;+ const requestedResults = Math.max(options.num_results, 0);+ const startPage = options.page ?? 1;const url = config.SEARXNG_ENDPOINT!;- // Remove trailing slash if it existsconst cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;-- // Concatenate "/search" to the cleaned URLconst finalUrl = cleanedUrl + "/search";- try {+ const fetchPage = async (page: number): Promise<SearchResult[]> => {+ const params = {+ q: q,+ language: options.lang,+ // gl: options.country, //not possible with SearXNG+ // location: options.location, //not possible with SearXNG+ // num: options.num_results, //not possible with SearXNG+ engines: config.SEARXNG_ENGINES ?? "",+ categories: config.SEARXNG_CATEGORIES ?? "",+ pageno: page,+ format: "json",+ };+const response = await axios.get(finalUrl, {headers: {"Content-Type": "application/json",@@ -52,9 +53,34 @@ export async function searxng_search(title: a.title,description: a.content,}));- } else {+ }++ return [];+ };++ try {+ if (requestedResults === 0) {return [];}++ const pagesToFetch = Math.max(+ 1,+ Math.ceil(requestedResults / resultsPerPage),+ );+ let results: SearchResult[] = [];++ for (let pageOffset = 0; pageOffset < pagesToFetch; pageOffset += 1) {+ const pageResults = await fetchPage(startPage + pageOffset);+ if (pageResults.length === 0) {+ break;+ }+ results = results.concat(pageResults);+ if (results.length >= requestedResults) {+ break;+ }+ }++ return results.slice(0, requestedResults);} catch (error) {logger.error(`There was an error searching for content`, { error });return [];diff --git a/apps/api/src/search/v2/searxng.ts b/apps/api/src/search/v2/searxng.tsindex 99af9f9ea..445c007ed 100644--- a/apps/api/src/search/v2/searxng.ts+++ b/apps/api/src/search/v2/searxng.ts@@ -17,26 +17,27 @@ export async function searxng_search(q: string,options: SearchOptions,): Promise<SearchV2Response> {- const params = {- q: q,- language: options.lang,- // gl: options.country, //not possible with SearXNG- // location: options.location, //not possible with SearXNG- // num: options.num_results, //not possible with SearXNG- engines: config.SEARXNG_ENGINES ?? "",- categories: config.SEARXNG_CATEGORIES ?? "",- pageno: options.page ?? 1,- format: "json",- };+ const resultsPerPage = 20;+ const requestedResults = Math.max(options.num_results, 0);+ const startPage = options.page ?? 1;const url = config.SEARXNG_ENDPOINT!;- // Remove trailing slash if it existsconst cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;-- // Concatenate "/search" to the cleaned URLconst finalUrl = cleanedUrl + "/search";- try {+ const fetchPage = async (page: number): Promise<WebSearchResult[]> => {+ const params = {+ q: q,+ language: options.lang,+ // gl: options.country, //not possible with SearXNG+ // location: options.location, //not possible with SearXNG+ // num: options.num_results, //not possible with SearXNG+ engines: config.SEARXNG_ENGINES ?? "",+ categories: config.SEARXNG_CATEGORIES ?? "",+ pageno: page,+ format: "json",+ };+const response = await axios.get(finalUrl, {headers: {"Content-Type": "application/json",@@ -47,18 +48,43 @@ export async function searxng_search(const data = response.data;if (data && Array.isArray(data.results)) {- const webResults: WebSearchResult[] = data.results.map((a: any) => ({+ return data.results.map((a: any) => ({url: a.url,title: a.title,description: a.content,}));+ }++ return [];+ };- return {- web: webResults,- };- } else {+ try {+ if (requestedResults === 0) {return {};}++ const pagesToFetch = Math.max(+ 1,+ Math.ceil(requestedResults / resultsPerPage),+ );+ let webResults: WebSearchResult[] = [];++ for (let pageOffset = 0; pageOffset < pagesToFetch; pageOffset += 1) {+ const pageResults = await fetchPage(startPage + pageOffset);+ if (pageResults.length === 0) {+ break;+ }+ webResults = webResults.concat(pageResults);+ if (webResults.length >= requestedResults) {+ break;+ }+ }++ return webResults.length > 0+ ? {+ web: webResults.slice(0, requestedResults),+ }+ : {};} catch (error) {logger.error(`There was an error searching for content`, { error });return {};