An open source, non-profit web search engine
"""
Investigate adding term information to the database.
How much extra space will it take?
"""
from pathlib import Path
from random import Random
import numpy as np
from scipy.stats import sem
from zstandard import ZstdCompressor
from mwmbl.tinysearchengine.indexer import Document, TinyIndex, _trim_items_to_page
from mwmbl.utils import add_term_info
random = Random(1)
INDEX_PATH = Path(__file__).parent.parent / "devdata" / "index-v2.tinysearch"
def run():
compressor = ZstdCompressor()
with TinyIndex(Document, INDEX_PATH) as index:
# Get some random integers between 0 and index.num_pages:
pages = random.sample(range(index.num_pages), 10000)
old_sizes = []
new_sizes = []
for i in pages:
page = index.get_page(i)
term_documents = []
for document in page:
term_document = add_term_info(document, index, i)
term_documents.append(term_document)
value_tuples = [document.as_tuple() for document in term_documents]
num_fitting, compressed = _trim_items_to_page(compressor, index.page_size, value_tuples)
new_sizes.append(num_fitting)
old_sizes.append(len(page))
print("Old sizes mean", np.mean(old_sizes), sem(old_sizes))
print("New sizes mean", np.mean(new_sizes), sem(new_sizes))
if __name__ == "__main__":
run()