Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Be more sophisticated about how pages are ordered

Daoud Clarkeffaea432f5fb
diff --git a/mwmbl/format.py b/mwmbl/format.py
index 56df6d3..6ebc85b 100644
--- a/mwmbl/format.py
+++ b/mwmbl/format.py
@@ -1,9 +1,18 @@
import re
+from mwmbl.tinysearchengine.indexer import DocumentState
from mwmbl.tokenizer import tokenize, clean_unicode
-def format_result_with_pattern(pattern, result, source):
+DOCUMENT_SOURCES = {
+ DocumentState.FROM_GOOGLE: 'google',
+ DocumentState.FROM_USER: 'user',
+ DocumentState.VALIDATED: 'mwmbl',
+ DocumentState.CURATED: 'mwmbl',
+}
+
+
+def format_result_with_pattern(pattern, result):
formatted_result = {}
for content_type, content_raw in [('title', result.title), ('extract', result.extract)]:
content = clean_unicode(content_raw)
@@ -17,7 +26,7 @@ def format_result_with_pattern(pattern, result, source):
content_result.append({'value': content[start:end], 'is_bold': is_bold})
formatted_result[content_type] = content_result
formatted_result['url'] = result.url
- formatted_result['source'] = source
+ formatted_result['source'] = DOCUMENT_SOURCES[result.state] if result.state else 'mwmbl'
return formatted_result
@@ -35,8 +44,8 @@ def get_query_regex(terms, is_complete, is_url):
return pattern
-def format_result(result, query, source):
+def format_result(result, query):
tokens = tokenize(query)
pattern = get_query_regex(tokens, True, False)
- return format_result_with_pattern(pattern, result, source)
+ return format_result_with_pattern(pattern, result)
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index 4605247..33b0d02 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -33,6 +33,8 @@ def astuple(dc):
class DocumentState(IntEnum):
CURATED = 0
VALIDATED = 1
+ FROM_USER = 2
+ FROM_GOOGLE = 3
@dataclass
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 2e72477..9ccec5d 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -110,6 +110,15 @@ def order_results(terms: list[str], results: list[Document], is_complete: bool)
return filtered_results
+def deduplicate(results, seen_titles):
+ deduplicated_results = []
+ for result in results:
+ if result.title not in seen_titles:
+ deduplicated_results.append(result)
+ seen_titles.add(result.title)
+ return deduplicated_results
+
+
class Ranker:
def __init__(self, tiny_index: TinyIndex, completer: Completer):
self.tiny_index = tiny_index
@@ -119,20 +128,19 @@ class Ranker:
def order_results(self, terms, pages, is_complete):
pass
- def search(self, s: str, additional_results: list[Document], source: Optional[str] = None):
- mwmbl_results, terms, _ = self.get_results(s)
+ def search(self, s: str, additional_results: list[Document]):
+ results, terms, _ = self.get_results(s, additional_results)
is_complete = s.endswith(' ')
pattern = get_query_regex(terms, is_complete, False)
formatted_results = []
seen_urls = set()
- for results, source in [(additional_results, source), (mwmbl_results, 'mwmbl')]:
- for result in results:
- if result.url in seen_urls:
- continue
- formatted_result = format_result_with_pattern(pattern, result, source)
- formatted_results.append(formatted_result)
- seen_urls.add(result.url)
+ for result in results:
+ if result.url in seen_urls:
+ continue
+ formatted_result = format_result_with_pattern(pattern, result)
+ formatted_results.append(formatted_result)
+ seen_urls.add(result.url)
logger.info("Return results: %d", len(formatted_results))
return formatted_results
@@ -153,7 +161,7 @@ class Ranker:
completed = [' '.join(terms[:-1] + [t]) for t in adjusted_completions]
return [q, urls + completed]
- def get_results(self, q):
+ def get_results(self, q: str, additional_results: list[Document]):
terms = tokenize(q)
is_complete = q.endswith(' ')
@@ -169,27 +177,26 @@ class Ranker:
curation_items = self.tiny_index.retrieve(curation_term)
curated_items = [d for d in curation_items if d.state in {DocumentState.CURATED, DocumentState.VALIDATED}
and d.term == curation_term]
+
if len(curated_items) > 0:
- return curated_items, terms, completions
-
- bigrams = set(get_bigrams(len(terms), terms))
-
- pages = []
- seen_items = set()
- for term in retrieval_terms | bigrams:
- if term == curation_term:
- items = curation_items
- else:
- items = self.tiny_index.retrieve(term)
- if items is not None:
- for item in items:
- # if term in item.title.lower() or term in item.extract.lower():
- if item.title not in seen_items:
- pages.append(item)
- seen_items.add(item.title)
-
- ordered_results = self.order_results(terms, pages, is_complete)
- return ordered_results, terms, completions
+ deduplicated_additional = deduplicate(additional_results, {item.title for item in curated_items})
+ deduplicated_results = curated_items + deduplicated_additional
+ else:
+ bigrams = set(get_bigrams(len(terms), terms))
+
+ pages = []
+ for term in retrieval_terms | bigrams:
+ # An optimisation - we have already retrieved this, so make use of it
+ if term == curation_term:
+ items = curation_items
+ else:
+ items = self.tiny_index.retrieve(term)
+ if items is not None:
+ pages += items
+
+ ordered_results = self.order_results(terms, pages + additional_results, is_complete)
+ deduplicated_results = deduplicate(ordered_results, set())
+ return deduplicated_results, terms, completions
class HeuristicRanker(Ranker):
diff --git a/mwmbl/views.py b/mwmbl/views.py
index 84b9c9f..de09661 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -18,7 +18,7 @@ from justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revis
STOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessor
from mwmbl.settings import NUM_EXTRACT_CHARS
-from mwmbl.tinysearchengine.indexer import Document
+from mwmbl.tinysearchengine.indexer import Document, DocumentState
from django.conf import settings
@@ -90,17 +90,18 @@ def _get_results_and_activity(request):
if query:
# There may be extra results in the request that we need to add in
# format is ?enhanced=google&title=title1&url=url1&extract=extract1&title=title2&url=url2&extract=extract2
- source = request.GET.get("enhanced", "unknown")
+ # source = request.GET.get("enhanced", "unknown")
titles = request.GET.getlist(f"title")
urls = request.GET.getlist(f"url")
extracts = request.GET.getlist(f"extract")
+ # For now, we only support the Google source
additional_results = [
- Document(title=title, url=url, extract=extract, score=0.0)
+ Document(title=title, url=url, extract=extract, score=0.0, state=DocumentState.FROM_GOOGLE)
for title, url, extract in zip(titles, urls, extracts)
]
- results = ranker.search(query, additional_results=additional_results, source=source)
+ results = ranker.search(query, additional_results=additional_results)
activity = None
else:
results = None
@@ -135,7 +136,7 @@ def fetch_url(request):
if len(extract) > NUM_EXTRACT_CHARS:
extract = extract[:NUM_EXTRACT_CHARS - 1] + '…'
- result = Document(title=title, url=url, extract=extract, score=0.0)
+ result = Document(title=title, url=url, extract=extract, score=0.0, state=DocumentState.FROM_USER)
return render(request, "result.html", {
- "result": format_result(result, query, "user"),
+ "result": format_result(result, query),
})