Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Extract highlighted terms

Daoud Clarke67b023b5cebe
diff --git a/mwmbl/format.py b/mwmbl/format.py
index fba55ae..1696fc0 100644
--- a/mwmbl/format.py
+++ b/mwmbl/format.py
@@ -1,5 +1,4 @@
import re
-from urllib.parse import urlparse
from mwmbl.tinysearchengine.indexer import DocumentState
from mwmbl.tokenizer import tokenize, clean_unicode
@@ -78,30 +77,40 @@ def format_result(result, query):
return format_result_with_pattern(pattern, result)
-def format_result_v2(result, position: int) -> dict:
- parsed = urlparse(result.url)
- source = get_document_source(result.state)
+def _extract_highlights(segments: list[dict]) -> list[str]:
+ """Merge consecutive bold segments (with whitespace-only gaps) into phrases."""
+ phrases = []
+ current: list[str] = []
+ for seg in segments:
+ if seg['is_bold']:
+ current.append(seg['value'])
+ elif current and not seg['value'].strip():
+ current.append(seg['value'])
+ else:
+ if current:
+ phrases.append(''.join(current).strip())
+ current = []
+ if current:
+ phrases.append(''.join(current).strip())
+
+ unique = set(phrases)
+ return sorted(unique, key=len, reverse=True)
+
+
+def format_result_v2(result, position: int, query: str) -> dict:
+ tokens = tokenize(query)
+ filtered_tokens = [t for t in tokens if t not in HIGHLIGHT_STOPWORDS]
+ pattern = get_query_regex(filtered_tokens, True, True)
+ v1 = format_result_with_pattern(pattern, result)
+ title = ''.join(seg['value'] for seg in v1['title'])
+ content = ''.join(seg['value'] for seg in v1['extract'])
return {
'url': result.url,
- 'title': clean_unicode(result.title) if result.title else '',
- 'content': clean_unicode(result.extract) if result.extract else '',
- 'engine': source,
- 'template': 'default.html',
- 'parsed_url': [
- parsed.scheme,
- parsed.netloc,
- parsed.path,
- parsed.params,
- parsed.query,
- parsed.fragment,
- ],
- 'img_src': '',
- 'thumbnail': '',
- 'priority': '',
- 'engines': [source],
+ 'title': title,
+ 'title_highlights': _extract_highlights(v1['title']),
+ 'content': content,
+ 'content_highlights': _extract_highlights(v1['extract']),
+ 'engine': get_document_source(result.state),
'positions': [position],
'score': 1.0 / position,
- 'category': 'general',
- 'publishedDate': None,
}
-
diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.py
index 0eaa9ed..7f15cf6 100644
--- a/mwmbl/tinysearchengine/search.py
+++ b/mwmbl/tinysearchengine/search.py
@@ -74,16 +74,10 @@ class SearxResult(Schema):
title: str
content: str
engine: str
- template: str
- parsed_url: list[str]
- img_src: str
- thumbnail: str
- priority: str
- engines: list[str]
positions: list[int]
score: float
- category: str
- publishedDate: Optional[str] = None
+ title_highlights: list[str]
+ content_highlights: list[str]
class SearchResponse(Schema):
@@ -91,11 +85,6 @@ class SearchResponse(Schema):
query: str
number_of_results: int
results: list[SearxResult]
- answers: list[str] = []
- corrections: list[str] = []
- infoboxes: list = []
- suggestions: list[str] = []
- unresponsive_engines: list = []
monthly_usage: int | None = None
monthly_limit: int | None = None
@@ -333,7 +322,7 @@ def _register_search_v2(r: Router | NinjaAPI, ranker: HeuristicRanker):
monthly_usage = None
raw_results = ranker.search(q, [])
- formatted = [format_result_v2(r, i + 1) for i, r in enumerate(raw_results)]
+ formatted = [format_result_v2(r, i + 1, q) for i, r in enumerate(raw_results)]
return SearchResponse(
query=q,
number_of_results=len(formatted),