An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
Update to Searxng advanced search with crawling. Need further testing
casistackd8a1595b5fa7
diff --git a/.env.local.example b/.env.local.exampleindex 064b5394..e9a17917 100644--- a/.env.local.example+++ b/.env.local.example@@ -23,6 +23,8 @@ SEARXNG_PORT=8080 # default portSEARXNG_BIND_ADDRESS=0.0.0.0 # default addressSEARXNG_IMAGE_PROXY=true # enable image proxySEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP address+SEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced', only affects SearXNG searches+SEARXNG_MAX_RESULTS=10 # Maximum number of results to return from SearXNG# Optional# The settings below can be used optionally as needed.diff --git a/README.md b/README.mdindex be09c864..065c70a2 100644--- a/README.md+++ b/README.md@@ -32,7 +32,9 @@ An AI-powered search engine with a generative UI.- Specify the model to generate answers- Groq API support [※](https://github.com/miurla/morphic/pull/58)- Local Redis support-- SearXNG Search API support+- SearXNG Search API support with customizable depth (basic or advanced)+- Configurable search depth (basic or advanced)+- SearXNG Search API support with customizable depth## 🧱 Stack@@ -99,6 +101,18 @@ To use Upstash Redis:1. Set `USE_LOCAL_REDIS=false` or leave it unset in your `.env.local` file.2. Set `UPSTASH_REDIS_REST_URL` and `UPSTASH_REDIS_REST_TOKEN` with your Upstash credentials.++# SearXNG Configuration+SEARXNG_API_URL=http://localhost:8080 # Replace with your local SearXNG API URL or docker http://searxng:8080+SEARCH_API=tavily # use searxng, tavily or exa+SEARXNG_SECRET="" # generate a secret key e.g. openssl rand -base64 32+SEARXNG_PORT=8080 # default port+SEARXNG_BIND_ADDRESS=0.0.0.0 # default address+SEARXNG_IMAGE_PROXY=true # enable image proxy+SEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP address+SEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced', only affects SearXNG searches+SEARXNG_MAX_RESULTS=50 # Maximum number of results to return from SearXNG+```### 5. Run app locally@@ -171,6 +185,8 @@ Morphic now supports SearXNG as an alternative search backend. To use SearXNG:- SEARXNG_IMAGE_PROXY=true- SEARCH_API=searxng- SEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP+ - SEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced'+ - SEARXNG_MAX_RESULTS=50 # Maximum number of results to return from SearXNG3. Two configuration files are provided in the root directory:@@ -191,6 +207,7 @@ Morphic now supports SearXNG as an alternative search backend. To use SearXNG:- To disable the limiter entirely, set `LIMITER=false` in the SearXNG service environment variables.- For production use, consider adjusting the `SEARXNG_SECRET_KEY` to a secure, randomly generated value.- The `SEARXNG_IMAGE_PROXY` option allows SearXNG to proxy image results, enhancing privacy. Set to `true` to enable this feature.+- Set `SEARXNG_DEFAULT_DEPTH` to 'basic' or 'advanced' to control the default search depth.#### Troubleshootingdiff --git a/app/api/advanced-search/route.ts b/app/api/advanced-search/route.tsnew file mode 100644index 00000000..13f30dbc--- /dev/null+++ b/app/api/advanced-search/route.ts@@ -0,0 +1,231 @@+import { NextResponse } from 'next/server'+import http from 'http'+import https from 'https'+import { parse } from 'node-html-parser'+import {+ SearchResults,+ SearXNGResponse,+ SearXNGResult,+ SearchResultItem+} from '@/lib/types'+import { Agent } from 'http'++export async function POST(request: Request) {+ const { query, maxResults, searchDepth, includeDomains, excludeDomains } =+ await request.json()++ try {+ const results = await advancedSearchXNGSearch(+ query,+ maxResults,+ searchDepth,+ includeDomains,+ excludeDomains+ )+ return NextResponse.json(results)+ } catch (error) {+ console.error('Advanced search error:', error)+ return NextResponse.json(+ { message: 'Internal Server Error' },+ { status: 500 }+ )+ }+}++async function advancedSearchXNGSearch(+ query: string,+ maxResults: number = 10,+ searchDepth: 'basic' | 'advanced' = 'advanced',+ includeDomains: string[] = [],+ excludeDomains: string[] = []+): Promise<SearchResults> {+ const apiUrl = process.env.SEARXNG_API_URL+ if (!apiUrl) {+ throw new Error('SEARXNG_API_URL is not set in the environment variables')+ }++ const maxAllowedResults = parseInt(process.env.SEARXNG_MAX_RESULTS || '50', 10)+ maxResults = Math.min(maxResults, maxAllowedResults)++ try {+ const url = new URL(`${apiUrl}/search`)+ url.searchParams.append('q', query)+ url.searchParams.append('format', 'json')+ url.searchParams.append('categories', 'general,images')+ url.searchParams.append('time_range', '')+ url.searchParams.append('safesearch', '0')+ url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')++ const resultsPerPage = 10+ const pageno = Math.ceil(maxResults / resultsPerPage)+ url.searchParams.append('pageno', String(pageno))++ // Note: includeDomains and excludeDomains are not used in the API call+ // but can be used for post-processing if needed++ const data: SearXNGResponse = await fetchJsonWithRetry(url.toString(), 3)++ let generalResults = data.results.filter(+ (result: SearXNGResult) => !result.img_src+ )++ // Apply domain filtering manually+ if (includeDomains.length > 0 || excludeDomains.length > 0) {+ generalResults = generalResults.filter(result => {+ const domain = new URL(result.url).hostname+ return (+ (includeDomains.length === 0 ||+ includeDomains.some(d => domain.includes(d))) &&+ (excludeDomains.length === 0 ||+ !excludeDomains.some(d => domain.includes(d)))+ )+ })+ }++ if (searchDepth === 'advanced') {+ const crawledResults = await Promise.allSettled(+ generalResults.slice(0, maxResults).map(result => crawlPage(result))+ )+ generalResults = crawledResults+ .filter(+ (result): result is PromiseFulfilledResult<SearXNGResult> =>+ result.status === 'fulfilled' && result.value !== null+ )+ .map(result => result.value)+ }++ generalResults = generalResults.slice(0, maxResults)++ const imageResults = data.results+ .filter((result: SearXNGResult) => result.img_src)+ .slice(0, maxResults)++ return {+ results: generalResults.map(+ (result: SearXNGResult): SearchResultItem => ({+ title: result.title,+ url: result.url,+ content: result.content+ })+ ),+ query: data.query,+ images: imageResults+ .map((result: SearXNGResult) => {+ const imgSrc = result.img_src || ''+ return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`+ })+ .filter(Boolean),+ number_of_results: data.number_of_results+ }+ } catch (error) {+ console.error('SearchXNG API error:', error)+ throw error+ }+}++async function crawlPage(result: SearXNGResult): Promise<SearXNGResult | null> {+ try {+ const html = await fetchHtmlWithTimeout(result.url, 10000) // 10 second timeout+ const root = parse(html)+ const body = root.querySelector('body')+ if (body) {+ const text = body.textContent.replace(/\s+/g, ' ').trim()+ const extractedText = text.substring(0, 2000)+ result.content = `${result.content}\n\nAdditional content:\n${extractedText}`+ }+ return result+ } catch (error) {+ console.error(`Error crawling ${result.url}:`, error)+ return null+ }+}++const httpAgent = new http.Agent({ keepAlive: true })+const httpsAgent = new https.Agent({ keepAlive: true })++async function fetchJsonWithRetry(url: string, retries: number): Promise<any> {+ for (let i = 0; i < retries; i++) {+ try {+ return await fetchJson(url)+ } catch (error) {+ if (i === retries - 1) throw error+ await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1)))+ }+ }+}++function fetchJson(url: string): Promise<any> {+ return new Promise((resolve, reject) => {+ const protocol = url.startsWith('https:') ? https : http+ const agent = url.startsWith('https:') ? httpsAgent : httpAgent+ const request = protocol.get(url, { agent }, res => {+ let data = ''+ res.on('data', chunk => {+ data += chunk+ })+ res.on('end', () => {+ try {+ resolve(JSON.parse(data))+ } catch (e) {+ reject(e)+ }+ })+ })+ request.on('error', reject)+ request.on('timeout', () => {+ request.destroy()+ reject(new Error('Request timed out'))+ })+ request.setTimeout(15000) // 15 second timeout+ })+}++async function fetchHtmlWithTimeout(+ url: string,+ timeoutMs: number+): Promise<string> {+ return Promise.race([+ fetchHtml(url),+ timeout(timeoutMs, `Fetching ${url} timed out after ${timeoutMs}ms`)+ ])+}++function fetchHtml(url: string): Promise<string> {+ return new Promise((resolve, reject) => {+ const protocol = url.startsWith('https:') ? https : http+ const agent = url.startsWith('https:') ? httpsAgent : httpAgent+ const request = protocol.get(url, { agent }, res => {+ if (+ res.statusCode &&+ res.statusCode >= 300 &&+ res.statusCode < 400 &&+ res.headers.location+ ) {+ // Handle redirects+ fetchHtml(new URL(res.headers.location, url).toString())+ .then(resolve)+ .catch(reject)+ return+ }+ let data = ''+ res.on('data', chunk => {+ data += chunk+ })+ res.on('end', () => resolve(data))+ })+ request.on('error', reject)+ request.on('timeout', () => {+ request.destroy()+ reject(new Error('Request timed out'))+ })+ request.setTimeout(5000) // 5 second timeout+ })+}++function timeout(ms: number, message: string): Promise<never> {+ return new Promise((_, reject) => {+ setTimeout(() => {+ reject(new Error(message))+ }, ms)+ })+}\ No newline at end of filediff --git a/bun.lockb b/bun.lockbindex 66c54b66..9fb4bfce 100755Binary files a/bun.lockb and b/bun.lockb differdiff --git a/lib/agents/tools/search.tsx b/lib/agents/tools/search.tsxindex 79129549..4c53538b 100644--- a/lib/agents/tools/search.tsx+++ b/lib/agents/tools/search.tsx@@ -40,20 +40,52 @@ export const searchTool = ({ uiStream, fullResponse }: ToolProps) =>let searchResult: SearchResultsconst searchAPI =(process.env.SEARCH_API as 'tavily' | 'exa' | 'searxng') || 'tavily'- console.log(`Using search API: ${searchAPI}`)+++ const effectiveSearchDepth =+ searchAPI === 'searxng' &&+ process.env.SEARXNG_DEFAULT_DEPTH === 'advanced'+ ? 'advanced'+ : search_depth || 'basic'++ console.log(+ `Using search API: ${searchAPI}, Search Depth: ${effectiveSearchDepth}`+ )try {- searchResult = await (searchAPI === 'tavily'- ? tavilySearch- : searchAPI === 'exa'- ? exaSearch- : searxngSearch)(- filledQuery,- max_results,- search_depth,- include_domains,- exclude_domains- )+ if (searchAPI === 'searxng' && effectiveSearchDepth === 'advanced') {+ // API route for advanced SearXNG search+ const baseUrl = process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000';+ const response = await fetch(`${baseUrl}/api/advanced-search`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({+ query: filledQuery,+ maxResults: max_results,+ searchDepth: effectiveSearchDepth,+ includeDomains: include_domains,+ excludeDomains: exclude_domains+ })+ })+ if (!response.ok) {+ throw new Error(+ `Advanced search API error: ${response.status} ${response.statusText}`+ )+ }+ searchResult = await response.json()+ } else {+ searchResult = await (searchAPI === 'tavily'+ ? tavilySearch+ : searchAPI === 'exa'+ ? exaSearch+ : searxngSearch)(+ filledQuery,+ max_results,+ effectiveSearchDepth,+ include_domains,+ exclude_domains+ )+ }} catch (error) {console.error('Search API error:', error)hasError = true@@ -171,9 +203,9 @@ async function exaSearch(async function searxngSearch(query: string,maxResults: number = 10,- _searchDepth: string,- includeDomains: string[] = [], //keep for future use- excludeDomains: string[] = [] //keep for future use+ searchDepth: string,+ includeDomains: string[] = [],+ excludeDomains: string[] = []): Promise<SearchResults> {const apiUrl = process.env.SEARXNG_API_URLif (!apiUrl) {@@ -185,9 +217,19 @@ async function searxngSearch(const url = new URL(`${apiUrl}/search`)url.searchParams.append('q', query)url.searchParams.append('format', 'json')- // Enable both general and image resultsurl.searchParams.append('categories', 'general,images')+ // Apply search depth settings+ if (searchDepth === 'advanced') {+ url.searchParams.append('time_range', '')+ url.searchParams.append('safesearch', '0')+ url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')+ } else {+ url.searchParams.append('time_range', 'year')+ url.searchParams.append('safesearch', '1')+ url.searchParams.append('engines', 'google,bing')+ }+// Fetch results from SearXNGconst response = await fetch(url.toString(), {method: 'GET',@@ -236,4 +278,4 @@ async function searxngSearch(console.error('SearXNG API error:', error)throw error}-}+}\ No newline at end of filediff --git a/next.config.js b/next.config.jsnew file mode 100644index 00000000..c3adedce--- /dev/null+++ b/next.config.js@@ -0,0 +1,12 @@+// /** @type {import('next').NextConfig} */+// const nextConfig = {+// experimental: {+// serverComponentsExternalPackages: ['node-html-parser']+// },+// env: {+// NEXT_PUBLIC_BASE_URL:+// process.env.NEXT_PUBLIC_BASE_URL || 'http://localhost:3000'+// }+// }++// module.exports = nextConfigdiff --git a/package.json b/package.jsonindex ab4a87ed..ed24b994 100644--- a/package.json+++ b/package.json@@ -36,6 +36,7 @@"lucide-react": "^0.363.0","next": "^14.2.3","next-themes": "^0.3.0",+ "node-html-parser": "^6.1.13","ollama-ai-provider": "^0.7.0","react-dom": "^18","react-icons": "^5.0.1",