Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Simplify old index

Daoud Clarkef96bd878ca6f
diff --git a/devdata/index-v2.tinysearch b/devdata/index-v2.tinysearch
index 5654547..19998b7 100644
Binary files a/devdata/index-v2.tinysearch and b/devdata/index-v2.tinysearch differ
diff --git a/mwmbl/background.py b/mwmbl/background.py
index da1e7d2..fdcd82b 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -15,9 +15,9 @@ from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.paths import BATCH_DIR_NAME, INDEX_NAME
from mwmbl.models import OldIndex
from mwmbl.tinysearchengine.copy_index import copy_pages
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document
-
-NUM_PAGES_TO_COPY = 10
+NUM_PAGES_TO_COPY = 1024
basicConfig(stream=sys.stdout, level=logging.INFO)
@@ -49,13 +49,13 @@ def copy_all_indexes(new_index_path):
# Check if all indexes are copied
+ num_updated = 0
for old_index_info in old_indexes:
- if old_index_info.last_page_copied >= old_index_info.num_pages - 1:
- logger.info(f"All pages copied for index {old_index_info.index_path}")
- continue
+ start_page = old_index_info.last_page_copied + 1 if old_index_info.last_page_copied else 0
+ end_page = copy_pages(old_index_info.index_path, new_index_path, start_page, NUM_PAGES_TO_COPY)
- logger.info(f"Copying pages from {old_index_info.index_path} to {new_index_path} starting at page {old_index_info.last_page_copied}")
- end_page = copy_pages(old_index_info.index_path, new_index_path, old_index_info.start_page, NUM_PAGES_TO_COPY)
+ if start_page == end_page:
+ continue
# Update the start page
old_index_info.last_page_copied = end_page
@@ -63,13 +63,18 @@ def copy_all_indexes(new_index_path):
old_index_info.save()
logger.info(f"Copied pages from {old_index_info.index_path} to {new_index_path} up to page {end_page}")
+ num_updated += 1
+ return num_updated
def copy_indexes_continuously():
new_index_path = Path(settings.DATA_PATH) / INDEX_NAME
while True:
+ num_updated = 0
try:
- copy_all_indexes(new_index_path)
+ num_updated = copy_all_indexes(new_index_path)
except Exception:
logger.exception("Error copying pages")
- sleep(10)
+
+ if num_updated == 0:
+ sleep(10)
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index b3838ac..74e0fb8 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -33,10 +33,14 @@ def run(batch_cache: BatchCache, index_path: str):
process_batch.run(batch_cache, BatchStatus.URLS_UPDATED, BatchStatus.INDEXED, process, 10000)
+def get_url_score(url):
+ # TODO: compute a proper score for each document
+ return 1/len(url)
+
+
def index_batches(batch_data: Collection[HashedBatch], index_path: str):
document_tuples = list(get_documents_from_batches(batch_data))
- # TODO: compute a proper score for each document
- documents = [Document(title, url, extract, 1/len(url)) for title, url, extract in document_tuples]
+ documents = [Document(title, url, extract, get_url_score(url)) for title, url, extract in document_tuples]
page_documents = preprocess_documents(documents, index_path)
index_pages(index_path, page_documents)
diff --git a/mwmbl/models.py b/mwmbl/models.py
index 327796f..abe79ce 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -30,6 +30,5 @@ class Curation(models.Model):
class OldIndex(models.Model):
index_path = models.CharField(max_length=300, primary_key=True)
- index_total_pages = models.IntegerField()
- last_copied_time = models.DateTimeField()
- last_page_copied = models.IntegerField()
+ last_copied_time = models.DateTimeField(null=True, blank=True)
+ last_page_copied = models.IntegerField(null=True, blank=True)
diff --git a/mwmbl/tinysearchengine/copy_index.py b/mwmbl/tinysearchengine/copy_index.py
index f940197..b8146c4 100644
--- a/mwmbl/tinysearchengine/copy_index.py
+++ b/mwmbl/tinysearchengine/copy_index.py
@@ -4,8 +4,9 @@ Copy an old index into a new one
from collections import defaultdict
from logging import getLogger
-from mwmbl.indexer.index_batches import index_pages
+from mwmbl.indexer.index_batches import index_pages, get_url_score
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+from mwmbl.utils import add_term_infos
logger = getLogger(__name__)
@@ -15,16 +16,26 @@ def copy_pages(old_index_path: str, new_index_path: str, start_page: int, num_pa
# Get all old indexes:
page_documents = defaultdict(list)
- with TinyIndex(item_factory=Document, index_path=new_index_path) as new_index:
+ with (TinyIndex(item_factory=Document, index_path=new_index_path) as new_index):
with TinyIndex(item_factory=Document, index_path=old_index_path) as old_index:
+ logger.info(f"Old index has {old_index.num_pages} pages")
+
# Copy each page in the old index into the new one
for page_index in range(start_page, start_page + num_pages_to_copy):
if page_index >= old_index.num_pages:
break
documents = old_index.get_page(page_index)
- assert all(document.term is not None for document in documents)
- for document in documents:
+ documents_with_terms = add_term_infos(documents, old_index, page_index)
+ documents_with_scores = [Document(
+ document.title,
+ document.url,
+ document.extract,
+ get_url_score(document.url),
+ document.term,
+ state=document.state
+ ) for document in documents_with_terms]
+ for document in documents_with_scores:
new_page = new_index.get_key_page_index(document.term)
page_documents[new_page].append(document)