An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
update search testing 6
casistack3dbf0d21cdd6
diff --git a/.env.local.example b/.env.local.exampleindex e9a17917..34088f40 100644--- a/.env.local.example+++ b/.env.local.example@@ -24,7 +24,10 @@ SEARXNG_BIND_ADDRESS=0.0.0.0 # default addressSEARXNG_IMAGE_PROXY=true # enable image proxySEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP addressSEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced', only affects SearXNG searches-SEARXNG_MAX_RESULTS=10 # Maximum number of results to return from SearXNG+SEARXNG_MAX_RESULTS=50 # Maximum number of results to return from SearXNG+SEARXNG_ENGINES=google,bing,duckduckgo,wikipedia # Search engines to use+SEARXNG_TIME_RANGE=None # Time range for search results: day, week, month, year, or None (for all time)+SEARXNG_SAFESEARCH=0 # Safe search setting: 0 (off), 1 (moderate), 2 (strict)# Optional# The settings below can be used optionally as needed.@@ -61,4 +64,6 @@ SEARXNG_MAX_RESULTS=10 # Maximum number of results to return from SearXNG# enable the video search tool# Serper API Key retrieved here: https://serper.dev/api-key-# SERPER_API_KEY=[YOUR_SERPER_API_KEY]\ No newline at end of file+# SERPER_API_KEY=[YOUR_SERPER_API_KEY]++#NEXT_PUBLIC_BASE_URL=http://localhost:3000\ No newline at end of filediff --git a/README.md b/README.mdindex 065c70a2..85e24029 100644--- a/README.md+++ b/README.md@@ -174,11 +174,12 @@ This will allow you to use Morphic as your default search engine in the browser.### Using SearXNG as an Alternative Search Backend-Morphic now supports SearXNG as an alternative search backend. To use SearXNG:+Morphic now supports SearXNG as an alternative search backend with advanced search capabilities. To use SearXNG:1. Ensure you have Docker and Docker Compose installed on your system.2. In your `.env.local` file, set the following variables:+ - NEXT_PUBLIC_BASE_URL=http://localhost:3000 # Base URL for local development- SEARXNG_API_URL=http://localhost:8080 # Replace with your local SearXNG API URL or docker http://searxng:8080- SEARXNG_SECRET=your_secret_key_here- SEARXNG_PORT=8080@@ -187,28 +188,37 @@ Morphic now supports SearXNG as an alternative search backend. To use SearXNG:- SEARXNG_LIMITER=false # can be enabled to limit the number of requests per IP- SEARXNG_DEFAULT_DEPTH=basic # Set to 'basic' or 'advanced'- SEARXNG_MAX_RESULTS=50 # Maximum number of results to return from SearXNG+ - SEARXNG_ENGINES=google,bing,duckduckgo,wikipedia # can be overriden in searxng config+ - SEARXNG_TIME_RANGE= # Time range for search results+ - SEARXNG_SAFESEARCH=0 # Safe search setting+ - SEARXNG_CRAWL_MULTIPLIER=4 # Multiplier for the number of results to crawl in advanced search3. Two configuration files are provided in the root directory:-- `searxng-settings.yml`: This file contains the main configuration for SearXNG, including engine settings and server options.-- `searxng-limiter.toml`: This file configures the rate limiting and bot detection features of SearXNG.+ - `searxng-settings.yml`: This file contains the main configuration for SearXNG, including engine settings and server options.+ - `searxng-limiter.toml`: This file configures the rate limiting and bot detection features of SearXNG.4. Run `docker-compose up` to start the Morphic stack with SearXNG included.5. SearXNG will be available at `http://localhost:8080` and Morphic will use it as the search backend.+#### Advanced Search Configuration++- `NEXT_PUBLIC_BASE_URL`: Set this to your local development URL (http://localhost:3000) or your production URL when deploying.+- `SEARXNG_DEFAULT_DEPTH`: Set to 'basic' or 'advanced' to control the default search depth.+- `SEARXNG_MAX_RESULTS`: Maximum number of results to return from SearXNG.+- `SEARXNG_CRAWL_MULTIPLIER`: In advanced search mode, this multiplier determines how many results to crawl. For example, if `SEARXNG_MAX_RESULTS=10` and `SEARXNG_CRAWL_MULTIPLIER=4`, up to 40 results will be crawled before filtering and ranking.+- `SEARXNG_ENGINES`: Comma-separated list of search engines to use.+- `SEARXNG_TIME_RANGE`: Time range for search results (e.g., 'day', 'week', 'month', 'year', 'all').+- `SEARXNG_SAFESEARCH`: Safe search setting (0 for off, 1 for moderate, 2 for strict).++The advanced search feature includes content crawling, relevance scoring, and filtering to provide more accurate and comprehensive results.+#### Customizing SearXNG- You can modify `searxng-settings.yml` to enable/disable specific search engines, change UI settings, or adjust server options.- The `searxng-limiter.toml` file allows you to configure rate limiting and bot detection. This is useful if you're exposing SearXNG directly to the internet.- If you prefer not to use external configuration files, you can set these options using environment variables in the `docker-compose.yml` file or directly in the SearXNG container.-#### Advanced Configuration--- To disable the limiter entirely, set `LIMITER=false` in the SearXNG service environment variables.-- For production use, consider adjusting the `SEARXNG_SECRET_KEY` to a secure, randomly generated value.-- The `SEARXNG_IMAGE_PROXY` option allows SearXNG to proxy image results, enhancing privacy. Set to `true` to enable this feature.-- Set `SEARXNG_DEFAULT_DEPTH` to 'basic' or 'advanced' to control the default search depth.-#### Troubleshooting- If you encounter issues with specific search engines (e.g., Wikidata), you can disable them in `searxng-settings.yml`:diff --git a/app/api/advanced-search/route.ts b/app/api/advanced-search/route.tsindex 4dd665e0..d5e30aed 100644--- a/app/api/advanced-search/route.ts+++ b/app/api/advanced-search/route.ts@@ -14,19 +14,32 @@ export async function POST(request: Request) {const { query, maxResults, searchDepth, includeDomains, excludeDomains } =await request.json()+ const SEARXNG_MAX_RESULTS = parseInt(+ process.env.SEARXNG_MAX_RESULTS || '50',+ 10+ )+ const SEARXNG_DEFAULT_DEPTH = process.env.SEARXNG_DEFAULT_DEPTH || 'basic'+try {const results = await advancedSearchXNGSearch(query,- maxResults,- searchDepth,+ Math.min(maxResults, SEARXNG_MAX_RESULTS),+ searchDepth || SEARXNG_DEFAULT_DEPTH,includeDomains,excludeDomains)return NextResponse.json(results)} catch (error) {- //console.error('Advanced search error:', error)+ console.error('Advanced search error:', error)return NextResponse.json(- { message: 'Internal Server Error' },+ {+ message: 'Internal Server Error',+ error: error instanceof Error ? error.message : String(error),+ query: query,+ results: [],+ images: [],+ number_of_results: 0+ },{ status: 500 })}@@ -44,32 +57,44 @@ async function advancedSearchXNGSearch(throw new Error('SEARXNG_API_URL is not set in the environment variables')}- const maxAllowedResults = parseInt(- process.env.SEARXNG_MAX_RESULTS || '50',+ const SEARXNG_ENGINES =+ process.env.SEARXNG_ENGINES || 'google,bing,duckduckgo,wikipedia'+ const SEARXNG_TIME_RANGE = process.env.SEARXNG_TIME_RANGE || 'None'+ const SEARXNG_SAFESEARCH = process.env.SEARXNG_SAFESEARCH || '0'+ const SEARXNG_CRAWL_MULTIPLIER = parseInt(+ process.env.SEARXNG_CRAWL_MULTIPLIER || '4',10)- maxResults = Math.min(maxResults, maxAllowedResults)try {const url = new URL(`${apiUrl}/search`)url.searchParams.append('q', query)url.searchParams.append('format', 'json')url.searchParams.append('categories', 'general,images')- url.searchParams.append('time_range', '')- url.searchParams.append('safesearch', '0')- url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')++ // Add time_range if it's not 'None'+ if (SEARXNG_TIME_RANGE !== 'None') {+ url.searchParams.append('time_range', SEARXNG_TIME_RANGE)+ }++ url.searchParams.append('safesearch', SEARXNG_SAFESEARCH)+ url.searchParams.append('engines', SEARXNG_ENGINES)const resultsPerPage = 10const pageno = Math.ceil(maxResults / resultsPerPage)url.searchParams.append('pageno', String(pageno))- // Note: includeDomains and excludeDomains are not used in the API call- // but can be used for post-processing if needed+ console.log('SearXNG API URL:', url.toString()) // Log the full URL for debuggingconst data: SearXNGResponse = await fetchJsonWithRetry(url.toString(), 3)+ if (!data || !Array.isArray(data.results)) {+ console.error('Invalid response from SearXNG:', data)+ throw new Error('Invalid response from SearXNG')+ }+let generalResults = data.results.filter(- (result: SearXNGResult) => !result.img_src+ (result: SearXNGResult) => result && !result.img_src)// Apply domain filtering manually@@ -88,7 +113,7 @@ async function advancedSearchXNGSearch(if (searchDepth === 'advanced') {const crawledResults = await Promise.all(generalResults- .slice(0, maxResults * 4)+ .slice(0, maxResults * SEARXNG_CRAWL_MULTIPLIER).map(result => crawlPage(result, query)))generalResults = crawledResults@@ -108,30 +133,35 @@ async function advancedSearchXNGSearch(generalResults = generalResults.slice(0, maxResults)- const imageResults = data.results- .filter((result: SearXNGResult) => result.img_src)+ const imageResults = (data.results || [])+ .filter((result: SearXNGResult) => result && result.img_src).slice(0, maxResults)return {results: generalResults.map((result: SearXNGResult): SearchResultItem => ({- title: result.title,- url: result.url,- content: result.content+ title: result.title || '',+ url: result.url || '',+ content: result.content || ''})),- query: data.query,+ query: data.query || query,images: imageResults.map((result: SearXNGResult) => {const imgSrc = result.img_src || ''return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`}).filter(Boolean),- number_of_results: data.number_of_results+ number_of_results: data.number_of_results || generalResults.length}} catch (error) {console.error('SearchXNG API error:', error)- throw error+ return {+ results: [],+ query: query,+ images: [],+ number_of_results: 0+ }}}@@ -144,12 +174,8 @@ async function crawlPage(// virtual console to suppress JSDOM warningsconst virtualConsole = new VirtualConsole()- virtualConsole.on('error', () => {-- })- virtualConsole.on('warn', () => {-- })+ virtualConsole.on('error', () => {})+ virtualConsole.on('warn', () => {})const dom = new JSDOM(html, {runScripts: 'outside-only',@@ -442,12 +468,13 @@ function fetchHtml(url: string): Promise<string> {res.on('end', () => resolve(data))})request.on('error', error => {- console.error(`Error fetching ${url}:`, error)+ //console.error(`Error fetching ${url}:`, error)reject(error)})request.on('timeout', () => {request.destroy()- reject(new Error(`Request timed out for ${url}`))+ //reject(new Error(`Request timed out for ${url}`))+ resolve('')})request.setTimeout(10000) // 10 second timeout})