Skip to content

An AI-powered search engine with a generative UI

Update to Searxng advanced search with crawling. Need further testing

casistackd8a1595b5fa7
diff --git a/.env.local.example b/.env.local.example
index 064b5394..e9a17917 100644
--- a/.env.local.example
+++ b/.env.local.example
@@ -23,6 +23,8 @@ SEARXNG_PORT=8080 # default port
SEARXNG_BIND_ADDRESS=0.0.0.0 # default address
SEARXNG_IMAGE_PROXY=true # enable image proxy
SEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP address
+SEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced', only affects SearXNG searches
+SEARXNG_MAX_RESULTS=10 # Maximum number of results to return from SearXNG
# Optional
# The settings below can be used optionally as needed.
diff --git a/README.md b/README.md
index be09c864..065c70a2 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,9 @@ An AI-powered search engine with a generative UI.
- Specify the model to generate answers
- Groq API support [※](https://github.com/miurla/morphic/pull/58)
- Local Redis support
-- SearXNG Search API support
+- SearXNG Search API support with customizable depth (basic or advanced)
+- Configurable search depth (basic or advanced)
+- SearXNG Search API support with customizable depth
## 🧱 Stack
@@ -99,6 +101,18 @@ To use Upstash Redis:
1. Set `USE_LOCAL_REDIS=false` or leave it unset in your `.env.local` file.
2. Set `UPSTASH_REDIS_REST_URL` and `UPSTASH_REDIS_REST_TOKEN` with your Upstash credentials.
+
+# SearXNG Configuration
+SEARXNG_API_URL=http://localhost:8080 # Replace with your local SearXNG API URL or docker http://searxng:8080
+SEARCH_API=tavily # use searxng, tavily or exa
+SEARXNG_SECRET="" # generate a secret key e.g. openssl rand -base64 32
+SEARXNG_PORT=8080 # default port
+SEARXNG_BIND_ADDRESS=0.0.0.0 # default address
+SEARXNG_IMAGE_PROXY=true # enable image proxy
+SEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP address
+SEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced', only affects SearXNG searches
+SEARXNG_MAX_RESULTS=50 # Maximum number of results to return from SearXNG
+
```
### 5. Run app locally
@@ -171,6 +185,8 @@ Morphic now supports SearXNG as an alternative search backend. To use SearXNG:
- SEARXNG_IMAGE_PROXY=true
- SEARCH_API=searxng
- SEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP
+ - SEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced'
+ - SEARXNG_MAX_RESULTS=50 # Maximum number of results to return from SearXNG
3. Two configuration files are provided in the root directory:
@@ -191,6 +207,7 @@ Morphic now supports SearXNG as an alternative search backend. To use SearXNG:
- To disable the limiter entirely, set `LIMITER=false` in the SearXNG service environment variables.
- For production use, consider adjusting the `SEARXNG_SECRET_KEY` to a secure, randomly generated value.
- The `SEARXNG_IMAGE_PROXY` option allows SearXNG to proxy image results, enhancing privacy. Set to `true` to enable this feature.
+- Set `SEARXNG_DEFAULT_DEPTH` to 'basic' or 'advanced' to control the default search depth.
#### Troubleshooting
diff --git a/app/api/advanced-search/route.ts b/app/api/advanced-search/route.ts
new file mode 100644
index 00000000..13f30dbc
--- /dev/null
+++ b/app/api/advanced-search/route.ts
@@ -0,0 +1,231 @@
+import { NextResponse } from 'next/server'
+import http from 'http'
+import https from 'https'
+import { parse } from 'node-html-parser'
+import {
+ SearchResults,
+ SearXNGResponse,
+ SearXNGResult,
+ SearchResultItem
+} from '@/lib/types'
+import { Agent } from 'http'
+
+export async function POST(request: Request) {
+ const { query, maxResults, searchDepth, includeDomains, excludeDomains } =
+ await request.json()
+
+ try {
+ const results = await advancedSearchXNGSearch(
+ query,
+ maxResults,
+ searchDepth,
+ includeDomains,
+ excludeDomains
+ )
+ return NextResponse.json(results)
+ } catch (error) {
+ console.error('Advanced search error:', error)
+ return NextResponse.json(
+ { message: 'Internal Server Error' },
+ { status: 500 }
+ )
+ }
+}
+
+async function advancedSearchXNGSearch(
+ query: string,
+ maxResults: number = 10,
+ searchDepth: 'basic' | 'advanced' = 'advanced',
+ includeDomains: string[] = [],
+ excludeDomains: string[] = []
+): Promise<SearchResults> {
+ const apiUrl = process.env.SEARXNG_API_URL
+ if (!apiUrl) {
+ throw new Error('SEARXNG_API_URL is not set in the environment variables')
+ }
+
+ const maxAllowedResults = parseInt(process.env.SEARXNG_MAX_RESULTS || '50', 10)
+ maxResults = Math.min(maxResults, maxAllowedResults)
+
+ try {
+ const url = new URL(`${apiUrl}/search`)
+ url.searchParams.append('q', query)
+ url.searchParams.append('format', 'json')
+ url.searchParams.append('categories', 'general,images')
+ url.searchParams.append('time_range', '')
+ url.searchParams.append('safesearch', '0')
+ url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')
+
+ const resultsPerPage = 10
+ const pageno = Math.ceil(maxResults / resultsPerPage)
+ url.searchParams.append('pageno', String(pageno))
+
+ // Note: includeDomains and excludeDomains are not used in the API call
+ // but can be used for post-processing if needed
+
+ const data: SearXNGResponse = await fetchJsonWithRetry(url.toString(), 3)
+
+ let generalResults = data.results.filter(
+ (result: SearXNGResult) => !result.img_src
+ )
+
+ // Apply domain filtering manually
+ if (includeDomains.length > 0 || excludeDomains.length > 0) {
+ generalResults = generalResults.filter(result => {
+ const domain = new URL(result.url).hostname
+ return (
+ (includeDomains.length === 0 ||
+ includeDomains.some(d => domain.includes(d))) &&
+ (excludeDomains.length === 0 ||
+ !excludeDomains.some(d => domain.includes(d)))
+ )
+ })
+ }
+
+ if (searchDepth === 'advanced') {
+ const crawledResults = await Promise.allSettled(
+ generalResults.slice(0, maxResults).map(result => crawlPage(result))
+ )
+ generalResults = crawledResults
+ .filter(
+ (result): result is PromiseFulfilledResult<SearXNGResult> =>
+ result.status === 'fulfilled' && result.value !== null
+ )
+ .map(result => result.value)
+ }
+
+ generalResults = generalResults.slice(0, maxResults)
+
+ const imageResults = data.results
+ .filter((result: SearXNGResult) => result.img_src)
+ .slice(0, maxResults)
+
+ return {
+ results: generalResults.map(
+ (result: SearXNGResult): SearchResultItem => ({
+ title: result.title,
+ url: result.url,
+ content: result.content
+ })
+ ),
+ query: data.query,
+ images: imageResults
+ .map((result: SearXNGResult) => {
+ const imgSrc = result.img_src || ''
+ return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`
+ })
+ .filter(Boolean),
+ number_of_results: data.number_of_results
+ }
+ } catch (error) {
+ console.error('SearchXNG API error:', error)
+ throw error
+ }
+}
+
+async function crawlPage(result: SearXNGResult): Promise<SearXNGResult | null> {
+ try {
+ const html = await fetchHtmlWithTimeout(result.url, 10000) // 10 second timeout
+ const root = parse(html)
+ const body = root.querySelector('body')
+ if (body) {
+ const text = body.textContent.replace(/\s+/g, ' ').trim()
+ const extractedText = text.substring(0, 2000)
+ result.content = `${result.content}\n\nAdditional content:\n${extractedText}`
+ }
+ return result
+ } catch (error) {
+ console.error(`Error crawling ${result.url}:`, error)
+ return null
+ }
+}
+
+const httpAgent = new http.Agent({ keepAlive: true })
+const httpsAgent = new https.Agent({ keepAlive: true })
+
+async function fetchJsonWithRetry(url: string, retries: number): Promise<any> {
+ for (let i = 0; i < retries; i++) {
+ try {
+ return await fetchJson(url)
+ } catch (error) {
+ if (i === retries - 1) throw error
+ await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1)))
+ }
+ }
+}
+
+function fetchJson(url: string): Promise<any> {
+ return new Promise((resolve, reject) => {
+ const protocol = url.startsWith('https:') ? https : http
+ const agent = url.startsWith('https:') ? httpsAgent : httpAgent
+ const request = protocol.get(url, { agent }, res => {
+ let data = ''
+ res.on('data', chunk => {
+ data += chunk
+ })
+ res.on('end', () => {
+ try {
+ resolve(JSON.parse(data))
+ } catch (e) {
+ reject(e)
+ }
+ })
+ })
+ request.on('error', reject)
+ request.on('timeout', () => {
+ request.destroy()
+ reject(new Error('Request timed out'))
+ })
+ request.setTimeout(15000) // 15 second timeout
+ })
+}
+
+async function fetchHtmlWithTimeout(
+ url: string,
+ timeoutMs: number
+): Promise<string> {
+ return Promise.race([
+ fetchHtml(url),
+ timeout(timeoutMs, `Fetching ${url} timed out after ${timeoutMs}ms`)
+ ])
+}
+
+function fetchHtml(url: string): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const protocol = url.startsWith('https:') ? https : http
+ const agent = url.startsWith('https:') ? httpsAgent : httpAgent
+ const request = protocol.get(url, { agent }, res => {
+ if (
+ res.statusCode &&
+ res.statusCode >= 300 &&
+ res.statusCode < 400 &&
+ res.headers.location
+ ) {
+ // Handle redirects
+ fetchHtml(new URL(res.headers.location, url).toString())
+ .then(resolve)
+ .catch(reject)
+ return
+ }
+ let data = ''
+ res.on('data', chunk => {
+ data += chunk
+ })
+ res.on('end', () => resolve(data))
+ })
+ request.on('error', reject)
+ request.on('timeout', () => {
+ request.destroy()
+ reject(new Error('Request timed out'))
+ })
+ request.setTimeout(5000) // 5 second timeout
+ })
+}
+
+function timeout(ms: number, message: string): Promise<never> {
+ return new Promise((_, reject) => {
+ setTimeout(() => {
+ reject(new Error(message))
+ }, ms)
+ })
+}
\ No newline at end of file
diff --git a/bun.lockb b/bun.lockb
index 66c54b66..9fb4bfce 100755
Binary files a/bun.lockb and b/bun.lockb differ
diff --git a/lib/agents/tools/search.tsx b/lib/agents/tools/search.tsx
index 79129549..4c53538b 100644
--- a/lib/agents/tools/search.tsx
+++ b/lib/agents/tools/search.tsx
@@ -40,20 +40,52 @@ export const searchTool = ({ uiStream, fullResponse }: ToolProps) =>
let searchResult: SearchResults
const searchAPI =
(process.env.SEARCH_API as 'tavily' | 'exa' | 'searxng') || 'tavily'
- console.log(`Using search API: ${searchAPI}`)
+
+
+ const effectiveSearchDepth =
+ searchAPI === 'searxng' &&
+ process.env.SEARXNG_DEFAULT_DEPTH === 'advanced'
+ ? 'advanced'
+ : search_depth || 'basic'
+
+ console.log(
+ `Using search API: ${searchAPI}, Search Depth: ${effectiveSearchDepth}`
+ )
try {
- searchResult = await (searchAPI === 'tavily'
- ? tavilySearch
- : searchAPI === 'exa'
- ? exaSearch
- : searxngSearch)(
- filledQuery,
- max_results,
- search_depth,
- include_domains,
- exclude_domains
- )
+ if (searchAPI === 'searxng' && effectiveSearchDepth === 'advanced') {
+ // API route for advanced SearXNG search
+ const baseUrl = process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000';
+ const response = await fetch(`${baseUrl}/api/advanced-search`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ query: filledQuery,
+ maxResults: max_results,
+ searchDepth: effectiveSearchDepth,
+ includeDomains: include_domains,
+ excludeDomains: exclude_domains
+ })
+ })
+ if (!response.ok) {
+ throw new Error(
+ `Advanced search API error: ${response.status} ${response.statusText}`
+ )
+ }
+ searchResult = await response.json()
+ } else {
+ searchResult = await (searchAPI === 'tavily'
+ ? tavilySearch
+ : searchAPI === 'exa'
+ ? exaSearch
+ : searxngSearch)(
+ filledQuery,
+ max_results,
+ effectiveSearchDepth,
+ include_domains,
+ exclude_domains
+ )
+ }
} catch (error) {
console.error('Search API error:', error)
hasError = true
@@ -171,9 +203,9 @@ async function exaSearch(
async function searxngSearch(
query: string,
maxResults: number = 10,
- _searchDepth: string,
- includeDomains: string[] = [], //keep for future use
- excludeDomains: string[] = [] //keep for future use
+ searchDepth: string,
+ includeDomains: string[] = [],
+ excludeDomains: string[] = []
): Promise<SearchResults> {
const apiUrl = process.env.SEARXNG_API_URL
if (!apiUrl) {
@@ -185,9 +217,19 @@ async function searxngSearch(
const url = new URL(`${apiUrl}/search`)
url.searchParams.append('q', query)
url.searchParams.append('format', 'json')
- // Enable both general and image results
url.searchParams.append('categories', 'general,images')
+ // Apply search depth settings
+ if (searchDepth === 'advanced') {
+ url.searchParams.append('time_range', '')
+ url.searchParams.append('safesearch', '0')
+ url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')
+ } else {
+ url.searchParams.append('time_range', 'year')
+ url.searchParams.append('safesearch', '1')
+ url.searchParams.append('engines', 'google,bing')
+ }
+
// Fetch results from SearXNG
const response = await fetch(url.toString(), {
method: 'GET',
@@ -236,4 +278,4 @@ async function searxngSearch(
console.error('SearXNG API error:', error)
throw error
}
-}
+}
\ No newline at end of file
diff --git a/next.config.js b/next.config.js
new file mode 100644
index 00000000..c3adedce
--- /dev/null
+++ b/next.config.js
@@ -0,0 +1,12 @@
+// /** @type {import('next').NextConfig} */
+// const nextConfig = {
+// experimental: {
+// serverComponentsExternalPackages: ['node-html-parser']
+// },
+// env: {
+// NEXT_PUBLIC_BASE_URL:
+// process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'
+// }
+// }
+
+// module.exports = nextConfig
diff --git a/package.json b/package.json
index ab4a87ed..ed24b994 100644
--- a/package.json
+++ b/package.json
@@ -36,6 +36,7 @@
"lucide-react": "^0.363.0",
"next": "^14.2.3",
"next-themes": "^0.3.0",
+ "node-html-parser": "^6.1.13",
"ollama-ai-provider": "^0.7.0",
"react-dom": "^18",
"react-icons": "^5.0.1",