Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

WIP: index pages from old index in new one

Daoud Clarke393ef7e06fa2
diff --git a/mwmbl/admin.py b/mwmbl/admin.py
index 74af7ac..a6027b0 100644
--- a/mwmbl/admin.py
+++ b/mwmbl/admin.py
@@ -2,7 +2,8 @@ from django.contrib.admin import ModelAdmin
from django.contrib.auth.admin import UserAdmin
from django.contrib import admin
-from mwmbl.models import MwmblUser, UserCuration
+from mwmbl.models import MwmblUser, UserCuration, OldIndex
admin.site.register(MwmblUser, UserAdmin)
admin.site.register(UserCuration, ModelAdmin)
+admin.site.register(OldIndex, ModelAdmin)
diff --git a/mwmbl/models.py b/mwmbl/models.py
index 7aec729..327796f 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -26,3 +26,10 @@ class Curation(models.Model):
original_results = models.JSONField()
new_results = models.JSONField()
num_changes = models.IntegerField(default=0)
+
+
+class OldIndex(models.Model):
+ index_path = models.CharField(max_length=300, primary_key=True)
+ index_total_pages = models.IntegerField()
+ last_copied_time = models.DateTimeField()
+ last_page_copied = models.IntegerField()
diff --git a/mwmbl/tinysearchengine/copy_index.py b/mwmbl/tinysearchengine/copy_index.py
new file mode 100644
index 0000000..33d251c
--- /dev/null
+++ b/mwmbl/tinysearchengine/copy_index.py
@@ -0,0 +1,31 @@
+"""
+Copy an old index into a new one
+"""
+from pathlib import Path
+
+from django.conf import settings
+
+from mwmbl.indexer.paths import INDEX_NAME
+from mwmbl.models import OldIndex
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+
+
+NUM_PAGES_TO_COPY = 10
+
+
+def copy_pages():
+ new_index_path = Path(settings.DATA_PATH) / INDEX_NAME
+
+ with TinyIndex(item_factory=Document, index_path=new_index_path) as new_index:
+ # Get all old indexes:
+ old_indexes = OldIndex.objects.all()
+ for old_index_info in old_indexes:
+ old_index_path = Path(old_index.index_path)
+ with TinyIndex(item_factory=Document, index_path=old_index_path) as old_index:
+ # Copy each page in the old index into the new one
+ start_page = old_index_info.start_page
+ for i in range(start_page, start_page + NUM_PAGES_TO_COPY):
+
+ # TODO: merge the items in the new and old indexes
+ page = old_index.get_page(i)
+ new_index.add_page(page)