An open source, non-profit web search engine
Sort documents using heuristic ranking
Daoud Clarke4ce36632f6a0
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex 0029acb..e1e0714 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -15,7 +15,7 @@ from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.index import tokenize_documentfrom mwmbl.indexer.indexdb import BatchStatusfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex-from mwmbl.tinysearchengine.rank import score_result, DOCUMENT_FREQUENCIES, N_DOCUMENTS+from mwmbl.tinysearchengine.rank import score_result, DOCUMENT_FREQUENCIES, N_DOCUMENTS, HeuristicRankerfrom mwmbl.utils import add_term_infoslogger = getLogger(__name__)@@ -45,7 +45,7 @@ def get_url_score(url):def index_batches(batch_data: Collection[HashedBatch], index_path: str):start_time = datetime.utcnow()document_tuples = list(get_documents_from_batches(batch_data))- documents = [Document(title, url, extract, 0.0) for title, url, extract in document_tuples]+ documents = [Document(title, url, extract) for title, url, extract in document_tuples]page_documents = preprocess_documents(documents, index_path)index_pages(index_path, page_documents)end_time = datetime.utcnow()@@ -54,12 +54,16 @@ def index_batches(batch_data: Collection[HashedBatch], index_path: str):def index_pages(index_path: str, page_documents: dict[int, list[Document]]):with TinyIndex(Document, index_path, 'w') as indexer:+ ranker = HeuristicRanker(indexer, None, score_threshold=float('-inf'))for page, documents in page_documents.items():new_documents = []existing_documents = indexer.get_page(page)seen_urls = set()seen_titles = set()- sorted_documents = sorted(documents + existing_documents, key=lambda x: x.score, reverse=True)++ # TODO: what about curated documents?+ sorted_documents = sort_documents(documents, existing_documents, ranker)+# TODO: for now we add the term here, until all the documents in the index have termssorted_documents_with_terms = add_term_infos(sorted_documents, indexer, page)for document in sorted_documents_with_terms:@@ -72,6 +76,34 @@ def index_pages(index_path: str, page_documents: dict[int, list[Document]]):indexer.store_in_page(page, new_documents)+def sort_documents(documents, all_existing_documents, ranker):+ curated_documents = [doc for doc in all_existing_documents if doc.state is not None]+ existing_documents = [doc for doc in all_existing_documents if doc.state is None]++ term_documents = defaultdict(list)++ for document in documents:+ if document.term is not None:+ term_documents[document.term].append(document)++ ordered_term_docs = defaultdict(list)+ for term, docs in term_documents.items():+ docs += [doc for doc in existing_documents if doc.term == term]+ ordered_docs = ranker.order_results(term.split(), docs, True)+ ordered_term_docs[term] = ordered_docs++ # Existing docs are already ordered+ other_terms = {doc.term for doc in existing_documents if doc.term not in ordered_term_docs}+ for doc in existing_documents:+ if doc.term in other_terms:+ ordered_term_docs[doc.term].append(doc)++ numbered_docs = [enumerate(docs) for docs in ordered_term_docs.values()]+ combined_docs = [doc for docs in numbered_docs for doc in docs]+ indexes, sorted_documents = zip(*sorted(combined_docs, key=lambda x: x[0]))+ return curated_documents + list(sorted_documents)++def preprocess_documents(documents, index_path):page_documents = defaultdict(list)with TinyIndex(Document, index_path, 'r') as indexer:@@ -81,44 +113,13 @@ def preprocess_documents(documents, index_path):tokenized = tokenize_document(document.url, document.title, document.extract, document.score)for token in tokenized.tokens:- score = score_document(token, document)- logger.info(f"Score for {repr(token)} in {document.url} with title {document.title}: {score}")page = indexer.get_key_page_index(token)- term_document = Document(document.title, document.url, document.extract, score, token)+ term_document = Document(document.title, document.url, document.extract, term=token)page_documents[page].append(term_document)print(f"Preprocessed for {len(page_documents)} pages")return page_documents-DOCUMENT_FREQ_DENOMINATOR = sum(DOCUMENT_FREQUENCIES.values()) / len(DOCUMENT_FREQUENCIES)---def round_sig(x, sig=2):- """- https://stackoverflow.com/a/3413529/660902- """- return round(x, sig - int(math.floor(math.log10(abs(x)))) - 1)---def score_document(token, document):- doc_score = score_result(token.split(), document, True) * 1000 + 1- # TODO: are we emphasising common words too much?- # It feels like we need something more like TF-IDF rather than just DF- token_score = get_token_score(token)- score = doc_score * token_score- rounded_score = round_sig(score)- if score > 10:- return int(rounded_score)- return rounded_score---def get_token_score(token):- terms = token.split()- doc_frequencies = [DOCUMENT_FREQUENCIES.get(term, 1) for term in terms]- doc_probs = [doc_freq/DOCUMENT_FREQ_DENOMINATOR for doc_freq in doc_frequencies]- return reduce(lambda x, y: x * y, doc_probs)--def get_url_error_status(item: Item):if item.status == 404:return URLStatus.ERROR_404diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 933ed37..49a60eb 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -49,11 +49,19 @@ class Document:title: strurl: strextract: str- score: float+ score: Optional[float] = Noneterm: Optional[str] = Nonestate: Optional[int] = None- def __init__(self, title, url, extract, score, term=None, state=None):+ def __init__(+ self,+ title: str,+ url: str,+ extract: str,+ score: Optional[float] = None,+ term: Optional[str] = None,+ state: Optional[int] = None+ ):# Sometimes the title or extract may be None, probably because of user generated content# It's not allowed to be None though, or things will breakself.title = title if title is not None else ''diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 2dba441..2540eec 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -220,7 +220,7 @@ class Ranker:self.completer = completer@abstractmethod- def order_results(self, terms, pages, is_complete):+ def order_results(self, terms: list[str], pages: list[Document], is_complete: bool):passdef search(self, s: str, additional_results: list[Document]) -> list[Document]:diff --git a/test/test_index_batches.py b/test/test_index_batches.pynew file mode 100644index 0000000..57fac78--- /dev/null+++ b/test/test_index_batches.py@@ -0,0 +1,64 @@+from mwmbl.indexer.index_batches import sort_documents+from mwmbl.tinysearchengine.indexer import Document, DocumentState+++class UrlRanker:+ @staticmethod+ def order_results(terms: list[str], pages: list[Document], is_complete: bool):+ return sorted(pages, key=lambda doc: doc.url)+++def test_sort_documents():+ existing_documents = [+ Document(title="title1", url="1", extract="extract1", term="term1"),+ Document(title="title2", url="4", extract="extract2", term="term2"),+ Document(title="title3", url="6", extract="extract3", term="term3"),+ Document(title="title4", url="5", extract="extract4", term="term3"),+ ]++ documents = [+ Document(title="title5", url="2", extract="extract5", term="term1"),+ Document(title="title6", url="3", extract="extract6", term="term2"),++ ]++ # Sort the documents+ sorted_documents = sort_documents(documents, existing_documents, UrlRanker())++ # Existing terms without new documents should not be sorted+ assert sorted_documents == [+ Document(title="title1", url="1", extract="extract1", term="term1"),+ Document(title="title6", url="3", extract="extract6", term="term2"),+ Document(title="title3", url="6", extract="extract3", term="term3"),+ Document(title="title5", url="2", extract="extract5", term="term1"),+ Document(title="title2", url="4", extract="extract2", term="term2"),+ Document(title="title4", url="5", extract="extract4", term="term3"),+ ]+++def test_sort_documents_curated_items_first():+ existing_documents = [+ Document(title="title1", url="1", extract="extract1", term="term1"),+ Document(title="title2", url="4", extract="extract2", term="term2"),+ Document(title="title3", url="6", extract="extract3", term="term1", state=DocumentState.ORGANIC_APPROVED),+ Document(title="title4", url="5", extract="extract4", term="term2", state=DocumentState.ORGANIC_APPROVED),+ ]++ documents = [+ Document(title="title5", url="2", extract="extract5", term="term1"),+ Document(title="title6", url="3", extract="extract6", term="term2"),++ ]++ # Sort the documents+ sorted_documents = sort_documents(documents, existing_documents, UrlRanker())++ # Curated items should be first+ assert sorted_documents == [+ Document(title="title3", url="6", extract="extract3", term="term1", state=DocumentState.ORGANIC_APPROVED),+ Document(title="title4", url="5", extract="extract4", term="term2", state=DocumentState.ORGANIC_APPROVED),+ Document(title="title1", url="1", extract="extract1", term="term1"),+ Document(title="title6", url="3", extract="extract6", term="term2"),+ Document(title="title5", url="2", extract="extract5", term="term1"),+ Document(title="title2", url="4", extract="extract2", term="term2"),+ ]