Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Store the best items, not the worst ones

Daoud Clarkeae658906dd82
diff --git a/analyse/inspect_index.py b/analyse/inspect_index.py
index 00c0273..2ebebe7 100644
--- a/analyse/inspect_index.py
+++ b/analyse/inspect_index.py
@@ -29,7 +29,7 @@ def store():
def get_items():
with TinyIndex(Document, INDEX_PATH) as tiny_index:
- items = tiny_index.retrieve('search')
+ items = tiny_index.retrieve('wikipedia')
if items:
for item in items:
print("Items", item)
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py
index f49afa4..02ef6f3 100644
--- a/mwmbl/crawler/urls.py
+++ b/mwmbl/crawler/urls.py
@@ -145,8 +145,6 @@ class URLDatabase:
return [result[0] for result in results]
-
-
def get_url_scores(self, urls: list[str]) -> dict[str, float]:
sql = f"""
SELECT url, score FROM urls WHERE url IN %(urls)s
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index eabc567..6b9b6e4 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -70,7 +70,7 @@ def index_pages(index_path, page_documents):
existing_documents = indexer.get_page(page)
seen_urls = set()
seen_titles = set()
- sorted_documents = sorted(documents + existing_documents, key=lambda x: x.score)
+ sorted_documents = sorted(documents + existing_documents, key=lambda x: x.score, reverse=True)
for document in sorted_documents:
if document.title in seen_titles or document.url in seen_urls:
continue
@@ -78,7 +78,6 @@ def index_pages(index_path, page_documents):
seen_urls.add(document.url)
seen_titles.add(document.title)
indexer.store_in_page(page, new_documents)
- logger.debug(f"Wrote page {page} with {len(new_documents)} documents")
def preprocess_documents(documents, index_path, nlp):