Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Handle the case where title or extract are None

Daoud Clarke67c519230e46
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 2cfdd2e..cd6ba3a 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -49,6 +49,7 @@ def score_match(last_match_char, match_length, total_possible_match_length):
def get_features(terms, title, url, extract, score, is_complete):
+ assert url is not None
features = {}
parsed_url = urlparse(url)
domain = parsed_url.netloc
@@ -58,8 +59,10 @@ def get_features(terms, title, url, extract, score, is_complete):
(domain, 'domain', True),
(domain, 'domain_tokenized', False),
(path, 'path', True)]:
+ part_not_none = part if part is not None else 'socks'
+ print("Part", part, name)
last_match_char, match_length, total_possible_match_length, match_terms = \
- get_match_features(terms, part, is_complete, is_url)
+ get_match_features(terms, part_not_none, is_complete, is_url)
features[f'last_match_char_{name}'] = last_match_char
features[f'match_length_{name}'] = match_length
features[f'total_possible_match_length_{name}'] = total_possible_match_length
@@ -82,11 +85,7 @@ def get_domain_score(url):
def get_match_features(terms, result_string, is_complete, is_url):
query_regex = get_query_regex(terms, is_complete, is_url)
- try:
- matches = list(re.finditer(query_regex, result_string, flags=re.IGNORECASE))
- except TypeError:
- logger.exception(f"Error getting match features, regex: {query_regex} result string: {result_string}")
- raise
+ matches = list(re.finditer(query_regex, result_string, flags=re.IGNORECASE))
# match_strings = {x.group(0).lower() for x in matches}
# match_length = sum(len(x) for x in match_strings)
diff --git a/test/test_rank.py b/test/test_rank.py
new file mode 100644
index 0000000..d26ca07
--- /dev/null
+++ b/test/test_rank.py
@@ -0,0 +1,27 @@
+from mwmbl.tinysearchengine.indexer import Document
+from mwmbl.tinysearchengine.rank import order_results
+
+
+def test_order_result():
+ doc1 = Document(title='title2', url='https://something.com', extract='extract2', score=2.0)
+ doc2 = Document(title='title3', url='https://something.com', extract='extract3', score=3.0)
+ doc3 = Document(title='Bananas and apples', url='https://something.com', extract='extract1', score=1.0)
+
+ documents = [doc1, doc2, doc3]
+
+ # Sort the documents
+ ordered_results = order_results(["bananas"], documents, True)
+
+ assert ordered_results[0].title == 'Bananas and apples'
+
+
+def test_order_results_where_document_has_properties_that_are_none():
+ doc1 = Document(title=None, url='https://something.com', extract='extract2', score=2.0)
+ doc2 = Document(title='Bananas and apples', url='https://something.com', extract=None, score=1.0)
+
+ documents = [doc1, doc2]
+
+ # Sort the documents
+ ordered_results = order_results(["bananas"], documents, True)
+
+ assert ordered_results[0].title == 'Bananas and apples'