An open source, non-profit web search engine
Improve result formatting
Daoud Clarke5a33b3ec490d
diff --git a/mwmbl/format.py b/mwmbl/format.pyindex 0a29fbc..6494adb 100644--- a/mwmbl/format.py+++ b/mwmbl/format.py@@ -15,6 +15,24 @@ DOCUMENT_SOURCES = {}+HIGHLIGHT_STOPWORDS = {+ # Articles & Determiners+ "a", "an", "the", "this", "that", "these", "those", "each", "every", "some", "any",+ # Prepositions+ "to", "in", "on", "at", "by", "for", "with", "about", "against", "between",+ "into", "through", "during", "before", "after", "above", "below", "from",+ "up", "down", "of", "off", "over", "under",+ # Conjunctions+ "and", "but", "or", "nor", "for", "yet", "so", "although", "because", "since", "unless",+ # Common Verbs & Pronouns+ "is", "am", "are", "was", "were", "be", "been", "being", "have", "has", "had",+ "do", "does", "did", "i", "me", "my", "you", "your", "he", "him", "his",+ "she", "her", "it", "its", "we", "us", "our", "they", "them", "their",+ # Interrogatives (usually noise in technical queries)+ "how", "what", "which", "who", "whom", "where", "when", "why"+}++def get_document_source(state: DocumentState):return DOCUMENT_SOURCES.get(state, 'mwmbl')@@ -30,18 +48,19 @@ def format_result_with_pattern(pattern, result):is_bold = i % 2 == 1start = all_spans[i]end = all_spans[i + 1]- content_result.append({'value': content[start:end], 'is_bold': is_bold})+ if end - start > 0:+ content_result.append({'value': content[start:end], 'is_bold': is_bold})formatted_result[content_type] = content_resultformatted_result['url'] = result.urlformatted_result['source'] = get_document_source(result.state)return formatted_result-def get_query_regex(terms, is_complete, is_url):+def get_query_regex(terms, is_complete: bool, use_word_boundaries: bool):if not terms:return ''- word_sep = r'\b' if is_url else ''+ word_sep = r'\b' if use_word_boundaries else ''if is_complete:term_patterns = [rf'{word_sep}{re.escape(term)}{word_sep}' for term in terms]else:@@ -53,6 +72,7 @@ def get_query_regex(terms, is_complete, is_url):def format_result(result, query):tokens = tokenize(query)- pattern = get_query_regex(tokens, True, False)+ filtered_tokens = [t for t in tokens if t not in HIGHLIGHT_STOPWORDS]+ pattern = get_query_regex(filtered_tokens, True, True)return format_result_with_pattern(pattern, result)