An open source, non-profit web search engine
Handle the case where title or extract are None
Daoud Clarke67c519230e46
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 2cfdd2e..cd6ba3a 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -49,6 +49,7 @@ def score_match(last_match_char, match_length, total_possible_match_length):def get_features(terms, title, url, extract, score, is_complete):+ assert url is not Nonefeatures = {}parsed_url = urlparse(url)domain = parsed_url.netloc@@ -58,8 +59,10 @@ def get_features(terms, title, url, extract, score, is_complete):(domain, 'domain', True),(domain, 'domain_tokenized', False),(path, 'path', True)]:+ part_not_none = part if part is not None else 'socks'+ print("Part", part, name)last_match_char, match_length, total_possible_match_length, match_terms = \- get_match_features(terms, part, is_complete, is_url)+ get_match_features(terms, part_not_none, is_complete, is_url)features[f'last_match_char_{name}'] = last_match_charfeatures[f'match_length_{name}'] = match_lengthfeatures[f'total_possible_match_length_{name}'] = total_possible_match_length@@ -82,11 +85,7 @@ def get_domain_score(url):def get_match_features(terms, result_string, is_complete, is_url):query_regex = get_query_regex(terms, is_complete, is_url)- try:- matches = list(re.finditer(query_regex, result_string, flags=re.IGNORECASE))- except TypeError:- logger.exception(f"Error getting match features, regex: {query_regex} result string: {result_string}")- raise+ matches = list(re.finditer(query_regex, result_string, flags=re.IGNORECASE))# match_strings = {x.group(0).lower() for x in matches}# match_length = sum(len(x) for x in match_strings)diff --git a/test/test_rank.py b/test/test_rank.pynew file mode 100644index 0000000..d26ca07--- /dev/null+++ b/test/test_rank.py@@ -0,0 +1,27 @@+from mwmbl.tinysearchengine.indexer import Document+from mwmbl.tinysearchengine.rank import order_results+++def test_order_result():+ doc1 = Document(title='title2', url='https://something.com', extract='extract2', score=2.0)+ doc2 = Document(title='title3', url='https://something.com', extract='extract3', score=3.0)+ doc3 = Document(title='Bananas and apples', url='https://something.com', extract='extract1', score=1.0)++ documents = [doc1, doc2, doc3]++ # Sort the documents+ ordered_results = order_results(["bananas"], documents, True)++ assert ordered_results[0].title == 'Bananas and apples'+++def test_order_results_where_document_has_properties_that_are_none():+ doc1 = Document(title=None, url='https://something.com', extract='extract2', score=2.0)+ doc2 = Document(title='Bananas and apples', url='https://something.com', extract=None, score=1.0)++ documents = [doc1, doc2]++ # Sort the documents+ ordered_results = order_results(["bananas"], documents, True)++ assert ordered_results[0].title == 'Bananas and apples'