An open source, non-profit web search engine
Extract highlighted terms
Daoud Clarke67b023b5cebe
diff --git a/mwmbl/format.py b/mwmbl/format.pyindex fba55ae..1696fc0 100644--- a/mwmbl/format.py+++ b/mwmbl/format.py@@ -1,5 +1,4 @@import re-from urllib.parse import urlparsefrom mwmbl.tinysearchengine.indexer import DocumentStatefrom mwmbl.tokenizer import tokenize, clean_unicode@@ -78,30 +77,40 @@ def format_result(result, query):return format_result_with_pattern(pattern, result)-def format_result_v2(result, position: int) -> dict:- parsed = urlparse(result.url)- source = get_document_source(result.state)+def _extract_highlights(segments: list[dict]) -> list[str]:+ """Merge consecutive bold segments (with whitespace-only gaps) into phrases."""+ phrases = []+ current: list[str] = []+ for seg in segments:+ if seg['is_bold']:+ current.append(seg['value'])+ elif current and not seg['value'].strip():+ current.append(seg['value'])+ else:+ if current:+ phrases.append(''.join(current).strip())+ current = []+ if current:+ phrases.append(''.join(current).strip())++ unique = set(phrases)+ return sorted(unique, key=len, reverse=True)+++def format_result_v2(result, position: int, query: str) -> dict:+ tokens = tokenize(query)+ filtered_tokens = [t for t in tokens if t not in HIGHLIGHT_STOPWORDS]+ pattern = get_query_regex(filtered_tokens, True, True)+ v1 = format_result_with_pattern(pattern, result)+ title = ''.join(seg['value'] for seg in v1['title'])+ content = ''.join(seg['value'] for seg in v1['extract'])return {'url': result.url,- 'title': clean_unicode(result.title) if result.title else '',- 'content': clean_unicode(result.extract) if result.extract else '',- 'engine': source,- 'template': 'default.html',- 'parsed_url': [- parsed.scheme,- parsed.netloc,- parsed.path,- parsed.params,- parsed.query,- parsed.fragment,- ],- 'img_src': '',- 'thumbnail': '',- 'priority': '',- 'engines': [source],+ 'title': title,+ 'title_highlights': _extract_highlights(v1['title']),+ 'content': content,+ 'content_highlights': _extract_highlights(v1['extract']),+ 'engine': get_document_source(result.state),'positions': [position],'score': 1.0 / position,- 'category': 'general',- 'publishedDate': None,}-diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.pyindex 0eaa9ed..7f15cf6 100644--- a/mwmbl/tinysearchengine/search.py+++ b/mwmbl/tinysearchengine/search.py@@ -74,16 +74,10 @@ class SearxResult(Schema):title: strcontent: strengine: str- template: str- parsed_url: list[str]- img_src: str- thumbnail: str- priority: str- engines: list[str]positions: list[int]score: float- category: str- publishedDate: Optional[str] = None+ title_highlights: list[str]+ content_highlights: list[str]class SearchResponse(Schema):@@ -91,11 +85,6 @@ class SearchResponse(Schema):query: strnumber_of_results: intresults: list[SearxResult]- answers: list[str] = []- corrections: list[str] = []- infoboxes: list = []- suggestions: list[str] = []- unresponsive_engines: list = []monthly_usage: int | None = Nonemonthly_limit: int | None = None@@ -333,7 +322,7 @@ def _register_search_v2(r: Router | NinjaAPI, ranker: HeuristicRanker):monthly_usage = Noneraw_results = ranker.search(q, [])- formatted = [format_result_v2(r, i + 1) for i, r in enumerate(raw_results)]+ formatted = [format_result_v2(r, i + 1, q) for i, r in enumerate(raw_results)]return SearchResponse(query=q,number_of_results=len(formatted),