An open source, non-profit web search engine
Be more sophisticated about how pages are ordered
Daoud Clarkeffaea432f5fb
diff --git a/mwmbl/format.py b/mwmbl/format.pyindex 56df6d3..6ebc85b 100644--- a/mwmbl/format.py+++ b/mwmbl/format.py@@ -1,9 +1,18 @@import re+from mwmbl.tinysearchengine.indexer import DocumentStatefrom mwmbl.tokenizer import tokenize, clean_unicode-def format_result_with_pattern(pattern, result, source):+DOCUMENT_SOURCES = {+ DocumentState.FROM_GOOGLE: 'google',+ DocumentState.FROM_USER: 'user',+ DocumentState.VALIDATED: 'mwmbl',+ DocumentState.CURATED: 'mwmbl',+}+++def format_result_with_pattern(pattern, result):formatted_result = {}for content_type, content_raw in [('title', result.title), ('extract', result.extract)]:content = clean_unicode(content_raw)@@ -17,7 +26,7 @@ def format_result_with_pattern(pattern, result, source):content_result.append({'value': content[start:end], 'is_bold': is_bold})formatted_result[content_type] = content_resultformatted_result['url'] = result.url- formatted_result['source'] = source+ formatted_result['source'] = DOCUMENT_SOURCES[result.state] if result.state else 'mwmbl'return formatted_result@@ -35,8 +44,8 @@ def get_query_regex(terms, is_complete, is_url):return pattern-def format_result(result, query, source):+def format_result(result, query):tokens = tokenize(query)pattern = get_query_regex(tokens, True, False)- return format_result_with_pattern(pattern, result, source)+ return format_result_with_pattern(pattern, result)diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 4605247..33b0d02 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -33,6 +33,8 @@ def astuple(dc):class DocumentState(IntEnum):CURATED = 0VALIDATED = 1+ FROM_USER = 2+ FROM_GOOGLE = 3@dataclassdiff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 2e72477..9ccec5d 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -110,6 +110,15 @@ def order_results(terms: list[str], results: list[Document], is_complete: bool)return filtered_results+def deduplicate(results, seen_titles):+ deduplicated_results = []+ for result in results:+ if result.title not in seen_titles:+ deduplicated_results.append(result)+ seen_titles.add(result.title)+ return deduplicated_results++class Ranker:def __init__(self, tiny_index: TinyIndex, completer: Completer):self.tiny_index = tiny_index@@ -119,20 +128,19 @@ class Ranker:def order_results(self, terms, pages, is_complete):pass- def search(self, s: str, additional_results: list[Document], source: Optional[str] = None):- mwmbl_results, terms, _ = self.get_results(s)+ def search(self, s: str, additional_results: list[Document]):+ results, terms, _ = self.get_results(s, additional_results)is_complete = s.endswith(' ')pattern = get_query_regex(terms, is_complete, False)formatted_results = []seen_urls = set()- for results, source in [(additional_results, source), (mwmbl_results, 'mwmbl')]:- for result in results:- if result.url in seen_urls:- continue- formatted_result = format_result_with_pattern(pattern, result, source)- formatted_results.append(formatted_result)- seen_urls.add(result.url)+ for result in results:+ if result.url in seen_urls:+ continue+ formatted_result = format_result_with_pattern(pattern, result)+ formatted_results.append(formatted_result)+ seen_urls.add(result.url)logger.info("Return results: %d", len(formatted_results))return formatted_results@@ -153,7 +161,7 @@ class Ranker:completed = [' '.join(terms[:-1] + [t]) for t in adjusted_completions]return [q, urls + completed]- def get_results(self, q):+ def get_results(self, q: str, additional_results: list[Document]):terms = tokenize(q)is_complete = q.endswith(' ')@@ -169,27 +177,26 @@ class Ranker:curation_items = self.tiny_index.retrieve(curation_term)curated_items = [d for d in curation_items if d.state in {DocumentState.CURATED, DocumentState.VALIDATED}and d.term == curation_term]+if len(curated_items) > 0:- return curated_items, terms, completions-- bigrams = set(get_bigrams(len(terms), terms))-- pages = []- seen_items = set()- for term in retrieval_terms | bigrams:- if term == curation_term:- items = curation_items- else:- items = self.tiny_index.retrieve(term)- if items is not None:- for item in items:- # if term in item.title.lower() or term in item.extract.lower():- if item.title not in seen_items:- pages.append(item)- seen_items.add(item.title)-- ordered_results = self.order_results(terms, pages, is_complete)- return ordered_results, terms, completions+ deduplicated_additional = deduplicate(additional_results, {item.title for item in curated_items})+ deduplicated_results = curated_items + deduplicated_additional+ else:+ bigrams = set(get_bigrams(len(terms), terms))++ pages = []+ for term in retrieval_terms | bigrams:+ # An optimisation - we have already retrieved this, so make use of it+ if term == curation_term:+ items = curation_items+ else:+ items = self.tiny_index.retrieve(term)+ if items is not None:+ pages += items++ ordered_results = self.order_results(terms, pages + additional_results, is_complete)+ deduplicated_results = deduplicate(ordered_results, set())+ return deduplicated_results, terms, completionsclass HeuristicRanker(Ranker):diff --git a/mwmbl/views.py b/mwmbl/views.pyindex 84b9c9f..de09661 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -18,7 +18,7 @@ from justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revisSTOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessorfrom mwmbl.settings import NUM_EXTRACT_CHARS-from mwmbl.tinysearchengine.indexer import Document+from mwmbl.tinysearchengine.indexer import Document, DocumentStatefrom django.conf import settings@@ -90,17 +90,18 @@ def _get_results_and_activity(request):if query:# There may be extra results in the request that we need to add in# format is ?enhanced=google&title=title1&url=url1&extract=extract1&title=title2&url=url2&extract=extract2- source = request.GET.get("enhanced", "unknown")+ # source = request.GET.get("enhanced", "unknown")titles = request.GET.getlist(f"title")urls = request.GET.getlist(f"url")extracts = request.GET.getlist(f"extract")+ # For now, we only support the Google sourceadditional_results = [- Document(title=title, url=url, extract=extract, score=0.0)+ Document(title=title, url=url, extract=extract, score=0.0, state=DocumentState.FROM_GOOGLE)for title, url, extract in zip(titles, urls, extracts)]- results = ranker.search(query, additional_results=additional_results, source=source)+ results = ranker.search(query, additional_results=additional_results)activity = Noneelse:results = None@@ -135,7 +136,7 @@ def fetch_url(request):if len(extract) > NUM_EXTRACT_CHARS:extract = extract[:NUM_EXTRACT_CHARS - 1] + '…'- result = Document(title=title, url=url, extract=extract, score=0.0)+ result = Document(title=title, url=url, extract=extract, score=0.0, state=DocumentState.FROM_USER)return render(request, "result.html", {- "result": format_result(result, query, "user"),+ "result": format_result(result, query),})