An open source, non-profit web search engine
Store the best items, not the worst ones
Daoud Clarkeae658906dd82
diff --git a/analyse/inspect_index.py b/analyse/inspect_index.pyindex 00c0273..2ebebe7 100644--- a/analyse/inspect_index.py+++ b/analyse/inspect_index.py@@ -29,7 +29,7 @@ def store():def get_items():with TinyIndex(Document, INDEX_PATH) as tiny_index:- items = tiny_index.retrieve('search')+ items = tiny_index.retrieve('wikipedia')if items:for item in items:print("Items", item)diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex f49afa4..02ef6f3 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -145,8 +145,6 @@ class URLDatabase:return [result[0] for result in results]--def get_url_scores(self, urls: list[str]) -> dict[str, float]:sql = f"""SELECT url, score FROM urls WHERE url IN %(urls)sdiff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex eabc567..6b9b6e4 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -70,7 +70,7 @@ def index_pages(index_path, page_documents):existing_documents = indexer.get_page(page)seen_urls = set()seen_titles = set()- sorted_documents = sorted(documents + existing_documents, key=lambda x: x.score)+ sorted_documents = sorted(documents + existing_documents, key=lambda x: x.score, reverse=True)for document in sorted_documents:if document.title in seen_titles or document.url in seen_urls:continue@@ -78,7 +78,6 @@ def index_pages(index_path, page_documents):seen_urls.add(document.url)seen_titles.add(document.title)indexer.store_in_page(page, new_documents)- logger.debug(f"Wrote page {page} with {len(new_documents)} documents")def preprocess_documents(documents, index_path, nlp):