Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Implement num_results for searxng (#2645)

Abimael Martell954cd969a413
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.ts
index e2f089f03..cbdfa67f4 100644
--- a/apps/api/src/__tests__/snips/v2/search.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search.test.ts
@@ -6,6 +6,7 @@ import {
TEST_PRODUCTION,
} from "../lib";
import { search, idmux, Identity } from "./lib";
+import { config } from "../../../config";
let identity: Identity;
@@ -214,4 +215,39 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("Search tests", () => {
},
60000,
);
+
+ // SEARXNG-specific pagination tests
+ concurrentIf(!!config.SEARXNG_ENDPOINT)(
+ "searxng respects limit of 2 results",
+ async () => {
+ const res = await search(
+ {
+ query: "firecrawl",
+ limit: 2,
+ },
+ identity,
+ );
+ expect(res.web).toBeDefined();
+ expect(res.web?.length).toBeGreaterThan(0);
+ expect(res.web?.length).toBeLessThanOrEqual(2);
+ },
+ 60000,
+ );
+
+ concurrentIf(!!config.SEARXNG_ENDPOINT)(
+ "searxng fetches multiple pages for 21 results",
+ async () => {
+ const res = await search(
+ {
+ query: "firecrawl",
+ limit: 21,
+ },
+ identity,
+ );
+ expect(res.web).toBeDefined();
+ expect(res.web?.length).toBeGreaterThan(0);
+ expect(res.web?.length).toBeLessThanOrEqual(21);
+ },
+ 60000,
+ );
});
diff --git a/apps/api/src/search/searxng.ts b/apps/api/src/search/searxng.ts
index 46017dfa0..74ab0f10a 100644
--- a/apps/api/src/search/searxng.ts
+++ b/apps/api/src/search/searxng.ts
@@ -17,26 +17,27 @@ export async function searxng_search(
q: string,
options: SearchOptions,
): Promise<SearchResult[]> {
- const params = {
- q: q,
- language: options.lang,
- // gl: options.country, //not possible with SearXNG
- // location: options.location, //not possible with SearXNG
- // num: options.num_results, //not possible with SearXNG
- engines: config.SEARXNG_ENGINES ?? "",
- categories: config.SEARXNG_CATEGORIES ?? "",
- pageno: options.page ?? 1,
- format: "json",
- };
+ const resultsPerPage = 20;
+ const requestedResults = Math.max(options.num_results, 0);
+ const startPage = options.page ?? 1;
const url = config.SEARXNG_ENDPOINT!;
- // Remove trailing slash if it exists
const cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;
-
- // Concatenate "/search" to the cleaned URL
const finalUrl = cleanedUrl + "/search";
- try {
+ const fetchPage = async (page: number): Promise<SearchResult[]> => {
+ const params = {
+ q: q,
+ language: options.lang,
+ // gl: options.country, //not possible with SearXNG
+ // location: options.location, //not possible with SearXNG
+ // num: options.num_results, //not possible with SearXNG
+ engines: config.SEARXNG_ENGINES ?? "",
+ categories: config.SEARXNG_CATEGORIES ?? "",
+ pageno: page,
+ format: "json",
+ };
+
const response = await axios.get(finalUrl, {
headers: {
"Content-Type": "application/json",
@@ -52,9 +53,34 @@ export async function searxng_search(
title: a.title,
description: a.content,
}));
- } else {
+ }
+
+ return [];
+ };
+
+ try {
+ if (requestedResults === 0) {
return [];
}
+
+ const pagesToFetch = Math.max(
+ 1,
+ Math.ceil(requestedResults / resultsPerPage),
+ );
+ let results: SearchResult[] = [];
+
+ for (let pageOffset = 0; pageOffset < pagesToFetch; pageOffset += 1) {
+ const pageResults = await fetchPage(startPage + pageOffset);
+ if (pageResults.length === 0) {
+ break;
+ }
+ results = results.concat(pageResults);
+ if (results.length >= requestedResults) {
+ break;
+ }
+ }
+
+ return results.slice(0, requestedResults);
} catch (error) {
logger.error(`There was an error searching for content`, { error });
return [];
diff --git a/apps/api/src/search/v2/searxng.ts b/apps/api/src/search/v2/searxng.ts
index 99af9f9ea..445c007ed 100644
--- a/apps/api/src/search/v2/searxng.ts
+++ b/apps/api/src/search/v2/searxng.ts
@@ -17,26 +17,27 @@ export async function searxng_search(
q: string,
options: SearchOptions,
): Promise<SearchV2Response> {
- const params = {
- q: q,
- language: options.lang,
- // gl: options.country, //not possible with SearXNG
- // location: options.location, //not possible with SearXNG
- // num: options.num_results, //not possible with SearXNG
- engines: config.SEARXNG_ENGINES ?? "",
- categories: config.SEARXNG_CATEGORIES ?? "",
- pageno: options.page ?? 1,
- format: "json",
- };
+ const resultsPerPage = 20;
+ const requestedResults = Math.max(options.num_results, 0);
+ const startPage = options.page ?? 1;
const url = config.SEARXNG_ENDPOINT!;
- // Remove trailing slash if it exists
const cleanedUrl = url.endsWith("/") ? url.slice(0, -1) : url;
-
- // Concatenate "/search" to the cleaned URL
const finalUrl = cleanedUrl + "/search";
- try {
+ const fetchPage = async (page: number): Promise<WebSearchResult[]> => {
+ const params = {
+ q: q,
+ language: options.lang,
+ // gl: options.country, //not possible with SearXNG
+ // location: options.location, //not possible with SearXNG
+ // num: options.num_results, //not possible with SearXNG
+ engines: config.SEARXNG_ENGINES ?? "",
+ categories: config.SEARXNG_CATEGORIES ?? "",
+ pageno: page,
+ format: "json",
+ };
+
const response = await axios.get(finalUrl, {
headers: {
"Content-Type": "application/json",
@@ -47,18 +48,43 @@ export async function searxng_search(
const data = response.data;
if (data && Array.isArray(data.results)) {
- const webResults: WebSearchResult[] = data.results.map((a: any) => ({
+ return data.results.map((a: any) => ({
url: a.url,
title: a.title,
description: a.content,
}));
+ }
+
+ return [];
+ };
- return {
- web: webResults,
- };
- } else {
+ try {
+ if (requestedResults === 0) {
return {};
}
+
+ const pagesToFetch = Math.max(
+ 1,
+ Math.ceil(requestedResults / resultsPerPage),
+ );
+ let webResults: WebSearchResult[] = [];
+
+ for (let pageOffset = 0; pageOffset < pagesToFetch; pageOffset += 1) {
+ const pageResults = await fetchPage(startPage + pageOffset);
+ if (pageResults.length === 0) {
+ break;
+ }
+ webResults = webResults.concat(pageResults);
+ if (webResults.length >= requestedResults) {
+ break;
+ }
+ }
+
+ return webResults.length > 0
+ ? {
+ web: webResults.slice(0, requestedResults),
+ }
+ : {};
} catch (error) {
logger.error(`There was an error searching for content`, { error });
return {};