An open source, non-profit web search engine
WIP: index pages from old index in new one
Daoud Clarke393ef7e06fa2
diff --git a/mwmbl/admin.py b/mwmbl/admin.pyindex 74af7ac..a6027b0 100644--- a/mwmbl/admin.py+++ b/mwmbl/admin.py@@ -2,7 +2,8 @@ from django.contrib.admin import ModelAdminfrom django.contrib.auth.admin import UserAdminfrom django.contrib import admin-from mwmbl.models import MwmblUser, UserCuration+from mwmbl.models import MwmblUser, UserCuration, OldIndexadmin.site.register(MwmblUser, UserAdmin)admin.site.register(UserCuration, ModelAdmin)+admin.site.register(OldIndex, ModelAdmin)diff --git a/mwmbl/models.py b/mwmbl/models.pyindex 7aec729..327796f 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -26,3 +26,10 @@ class Curation(models.Model):original_results = models.JSONField()new_results = models.JSONField()num_changes = models.IntegerField(default=0)+++class OldIndex(models.Model):+ index_path = models.CharField(max_length=300, primary_key=True)+ index_total_pages = models.IntegerField()+ last_copied_time = models.DateTimeField()+ last_page_copied = models.IntegerField()diff --git a/mwmbl/tinysearchengine/copy_index.py b/mwmbl/tinysearchengine/copy_index.pynew file mode 100644index 0000000..33d251c--- /dev/null+++ b/mwmbl/tinysearchengine/copy_index.py@@ -0,0 +1,31 @@+"""+Copy an old index into a new one+"""+from pathlib import Path++from django.conf import settings++from mwmbl.indexer.paths import INDEX_NAME+from mwmbl.models import OldIndex+from mwmbl.tinysearchengine.indexer import TinyIndex, Document+++NUM_PAGES_TO_COPY = 10+++def copy_pages():+ new_index_path = Path(settings.DATA_PATH) / INDEX_NAME++ with TinyIndex(item_factory=Document, index_path=new_index_path) as new_index:+ # Get all old indexes:+ old_indexes = OldIndex.objects.all()+ for old_index_info in old_indexes:+ old_index_path = Path(old_index.index_path)+ with TinyIndex(item_factory=Document, index_path=old_index_path) as old_index:+ # Copy each page in the old index into the new one+ start_page = old_index_info.start_page+ for i in range(start_page, start_page + NUM_PAGES_TO_COPY):++ # TODO: merge the items in the new and old indexes+ page = old_index.get_page(i)+ new_index.add_page(page)