An open source, non-profit web search engine
Fixes for API changes
Daoud Clarkeae3b334a7fce
diff --git a/mwmbl/indexer/index.py b/mwmbl/indexer/index.pyindex 94dc1e3..9368433 100644--- a/mwmbl/indexer/index.py+++ b/mwmbl/indexer/index.py@@ -10,7 +10,7 @@ import pandas as pd# NUM_PAGES = 8192# PAGE_SIZE = 512-from mwmbl.tinysearchengine.indexer import TinyIndexer, Document, TokenizedDocument+from mwmbl.tinysearchengine.indexer import Document, TokenizedDocument, TinyIndexHTTP_START = 'http://'HTTPS_START = 'https://'@@ -66,9 +66,7 @@ def grouper(n: int, iterator: Iterator):yield chunk-def index_titles_urls_and_extracts(indexer: TinyIndexer, nlp, titles_urls_and_extracts, terms_path):- indexer.create()-+def index_titles_urls_and_extracts(indexer: TinyIndex, nlp, titles_urls_and_extracts, terms_path):terms = Counter()pages = get_pages(nlp, titles_urls_and_extracts)for page in pages:diff --git a/mwmbl/indexer/index_crawl.py b/mwmbl/indexer/index_crawl.pyindex 3c30482..44e557e 100644--- a/mwmbl/indexer/index_crawl.py+++ b/mwmbl/indexer/index_crawl.py@@ -8,16 +8,18 @@ import spacyfrom mwmbl.indexer.fsqueue import FSQueue, GzipJsonBlobSerializer, FSQueueErrorfrom mwmbl.indexer.index import index_titles_urls_and_extractsfrom mwmbl.indexer.paths import INDEX_PATH, MWMBL_CRAWL_TERMS_PATH, DATA_DIR-from mwmbl.tinysearchengine.indexer import TinyIndexer, Document, NUM_PAGES, PAGE_SIZE+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, NUM_PAGES, PAGE_SIZElogger = getLogger(__name__)-def index_mwmbl_craw_data():+def index_mwmbl_crawl_data():nlp = spacy.load("en_core_web_sm")- with TinyIndexer(Document, INDEX_PATH, NUM_PAGES, PAGE_SIZE) as indexer:+ TinyIndex.create(Document, INDEX_PATH, NUM_PAGES, PAGE_SIZE)++ with TinyIndex(Document, INDEX_PATH, 'w') as indexer:titles_urls_and_extracts = get_mwmbl_crawl_titles_urls_and_extracts()index_titles_urls_and_extracts(indexer, nlp, titles_urls_and_extracts, MWMBL_CRAWL_TERMS_PATH)@@ -43,4 +45,4 @@ def get_mwmbl_crawl_titles_urls_and_extracts():if __name__ == '__main__':- index_mwmbl_craw_data()+ index_mwmbl_crawl_data()diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 5a33cb2..1beb1b0 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -9,8 +9,7 @@ import mmh3from zstandard import ZstdDecompressor, ZstdCompressorVERSION = 1-METADATA_CONSTANT = 'mwmbl-tiny-search'.encode('utf8')-METADATA_FORMAT = 'IIIs'+METADATA_CONSTANT = b'mwmbl-tiny-search'METADATA_SIZE = 4096NUM_PAGES = 76800@@ -117,17 +116,19 @@ class TinyIndex(Generic[T]):"""Get the page at index i, decompress and deserialise it using JSON"""+ results = self._get_page_tuples(i)+ return [self.item_factory(*item) for item in results]++ def _get_page_tuples(self, i):page_data = self.mmap[i * self.page_size:(i + 1) * self.page_size]decompressed_data = self.decompressor.decompress(page_data)- results = json.loads(decompressed_data.decode('utf8'))- converted = [self.item_factory(*item) for item in results]- return converted+ return json.loads(decompressed_data.decode('utf8'))def index(self, key: str, value: T):assert type(value) == self.item_factory, f"Can only index the specified type" \f" ({self.item_factory.__name__})"page_index = self._get_key_page_index(key)- current_page = self.get_page(page_index)+ current_page = self._get_page_tuples(page_index)if current_page is None:current_page = []value_tuple = astuple(value)@@ -151,7 +152,7 @@ class TinyIndex(Generic[T]):@staticmethoddef create(item_factory: Callable[..., T], index_path: str, num_pages: int, page_size: int):if os.path.isfile(index_path):- raise FileExistsError("Index file already exists")+ raise FileExistsError(f"Index file '{index_path}' already exists")metadata = TinyIndexMetadata(VERSION, page_size, num_pages, item_factory.__name__)metadata_bytes = metadata.to_bytes()