An open source, non-profit web search engine
Simplify old index
Daoud Clarkef96bd878ca6f
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearchindex 5654547..19998b7 100644Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differdiff --git a/mwmbl/background.py b/mwmbl/background.pyindex da1e7d2..fdcd82b 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -15,9 +15,9 @@ from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.paths import BATCH_DIR_NAME, INDEX_NAMEfrom mwmbl.models import OldIndexfrom mwmbl.tinysearchengine.copy_index import copy_pages+from mwmbl.tinysearchengine.indexer import TinyIndex, Document--NUM_PAGES_TO_COPY = 10+NUM_PAGES_TO_COPY = 1024basicConfig(stream=sys.stdout, level=logging.INFO)@@ -49,13 +49,13 @@ def copy_all_indexes(new_index_path):# Check if all indexes are copied+ num_updated = 0for old_index_info in old_indexes:- if old_index_info.last_page_copied >= old_index_info.num_pages - 1:- logger.info(f"All pages copied for index {old_index_info.index_path}")- continue+ start_page = old_index_info.last_page_copied + 1 if old_index_info.last_page_copied else 0+ end_page = copy_pages(old_index_info.index_path, new_index_path, start_page, NUM_PAGES_TO_COPY)- logger.info(f"Copying pages from {old_index_info.index_path} to {new_index_path} starting at page {old_index_info.last_page_copied}")- end_page = copy_pages(old_index_info.index_path, new_index_path, old_index_info.start_page, NUM_PAGES_TO_COPY)+ if start_page == end_page:+ continue# Update the start pageold_index_info.last_page_copied = end_page@@ -63,13 +63,18 @@ def copy_all_indexes(new_index_path):old_index_info.save()logger.info(f"Copied pages from {old_index_info.index_path} to {new_index_path} up to page {end_page}")+ num_updated += 1+ return num_updateddef copy_indexes_continuously():new_index_path = Path(settings.DATA_PATH) / INDEX_NAMEwhile True:+ num_updated = 0try:- copy_all_indexes(new_index_path)+ num_updated = copy_all_indexes(new_index_path)except Exception:logger.exception("Error copying pages")- sleep(10)++ if num_updated == 0:+ sleep(10)diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex b3838ac..74e0fb8 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -33,10 +33,14 @@ def run(batch_cache: BatchCache, index_path: str):process_batch.run(batch_cache, BatchStatus.URLS_UPDATED, BatchStatus.INDEXED, process, 10000)+def get_url_score(url):+ # TODO: compute a proper score for each document+ return 1/len(url)++def index_batches(batch_data: Collection[HashedBatch], index_path: str):document_tuples = list(get_documents_from_batches(batch_data))- # TODO: compute a proper score for each document- documents = [Document(title, url, extract, 1/len(url)) for title, url, extract in document_tuples]+ documents = [Document(title, url, extract, get_url_score(url)) for title, url, extract in document_tuples]page_documents = preprocess_documents(documents, index_path)index_pages(index_path, page_documents)diff --git a/mwmbl/models.py b/mwmbl/models.pyindex 327796f..abe79ce 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -30,6 +30,5 @@ class Curation(models.Model):class OldIndex(models.Model):index_path = models.CharField(max_length=300, primary_key=True)- index_total_pages = models.IntegerField()- last_copied_time = models.DateTimeField()- last_page_copied = models.IntegerField()+ last_copied_time = models.DateTimeField(null=True, blank=True)+ last_page_copied = models.IntegerField(null=True, blank=True)diff --git a/mwmbl/tinysearchengine/copy_index.py b/mwmbl/tinysearchengine/copy_index.pyindex f940197..b8146c4 100644--- a/mwmbl/tinysearchengine/copy_index.py+++ b/mwmbl/tinysearchengine/copy_index.py@@ -4,8 +4,9 @@ Copy an old index into a new onefrom collections import defaultdictfrom logging import getLogger-from mwmbl.indexer.index_batches import index_pages+from mwmbl.indexer.index_batches import index_pages, get_url_scorefrom mwmbl.tinysearchengine.indexer import TinyIndex, Document+from mwmbl.utils import add_term_infoslogger = getLogger(__name__)@@ -15,16 +16,26 @@ def copy_pages(old_index_path: str, new_index_path: str, start_page: int, num_pa# Get all old indexes:page_documents = defaultdict(list)- with TinyIndex(item_factory=Document, index_path=new_index_path) as new_index:+ with (TinyIndex(item_factory=Document, index_path=new_index_path) as new_index):with TinyIndex(item_factory=Document, index_path=old_index_path) as old_index:+ logger.info(f"Old index has {old_index.num_pages} pages")+# Copy each page in the old index into the new onefor page_index in range(start_page, start_page + num_pages_to_copy):if page_index >= old_index.num_pages:breakdocuments = old_index.get_page(page_index)- assert all(document.term is not None for document in documents)- for document in documents:+ documents_with_terms = add_term_infos(documents, old_index, page_index)+ documents_with_scores = [Document(+ document.title,+ document.url,+ document.extract,+ get_url_score(document.url),+ document.term,+ state=document.state+ ) for document in documents_with_terms]+ for document in documents_with_scores:new_page = new_index.get_key_page_index(document.term)page_documents[new_page].append(document)