Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Fixes for API changes

Daoud Clarkeae3b334a7fce
diff --git a/mwmbl/indexer/index.py b/mwmbl/indexer/index.py
index 94dc1e3..9368433 100644
--- a/mwmbl/indexer/index.py
+++ b/mwmbl/indexer/index.py
@@ -10,7 +10,7 @@ import pandas as pd
# NUM_PAGES = 8192
# PAGE_SIZE = 512
-from mwmbl.tinysearchengine.indexer import TinyIndexer, Document, TokenizedDocument
+from mwmbl.tinysearchengine.indexer import Document, TokenizedDocument, TinyIndex
HTTP_START = 'http://'
HTTPS_START = 'https://'
@@ -66,9 +66,7 @@ def grouper(n: int, iterator: Iterator):
yield chunk
-def index_titles_urls_and_extracts(indexer: TinyIndexer, nlp, titles_urls_and_extracts, terms_path):
- indexer.create()
-
+def index_titles_urls_and_extracts(indexer: TinyIndex, nlp, titles_urls_and_extracts, terms_path):
terms = Counter()
pages = get_pages(nlp, titles_urls_and_extracts)
for page in pages:
diff --git a/mwmbl/indexer/index_crawl.py b/mwmbl/indexer/index_crawl.py
index 3c30482..44e557e 100644
--- a/mwmbl/indexer/index_crawl.py
+++ b/mwmbl/indexer/index_crawl.py
@@ -8,16 +8,18 @@ import spacy
from mwmbl.indexer.fsqueue import FSQueue, GzipJsonBlobSerializer, FSQueueError
from mwmbl.indexer.index import index_titles_urls_and_extracts
from mwmbl.indexer.paths import INDEX_PATH, MWMBL_CRAWL_TERMS_PATH, DATA_DIR
-from mwmbl.tinysearchengine.indexer import TinyIndexer, Document, NUM_PAGES, PAGE_SIZE
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, NUM_PAGES, PAGE_SIZE
logger = getLogger(__name__)
-def index_mwmbl_craw_data():
+def index_mwmbl_crawl_data():
nlp = spacy.load("en_core_web_sm")
- with TinyIndexer(Document, INDEX_PATH, NUM_PAGES, PAGE_SIZE) as indexer:
+ TinyIndex.create(Document, INDEX_PATH, NUM_PAGES, PAGE_SIZE)
+
+ with TinyIndex(Document, INDEX_PATH, 'w') as indexer:
titles_urls_and_extracts = get_mwmbl_crawl_titles_urls_and_extracts()
index_titles_urls_and_extracts(indexer, nlp, titles_urls_and_extracts, MWMBL_CRAWL_TERMS_PATH)
@@ -43,4 +45,4 @@ def get_mwmbl_crawl_titles_urls_and_extracts():
if __name__ == '__main__':
- index_mwmbl_craw_data()
+ index_mwmbl_crawl_data()
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index 5a33cb2..1beb1b0 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -9,8 +9,7 @@ import mmh3
from zstandard import ZstdDecompressor, ZstdCompressor
VERSION = 1
-METADATA_CONSTANT = 'mwmbl-tiny-search'.encode('utf8')
-METADATA_FORMAT = 'IIIs'
+METADATA_CONSTANT = b'mwmbl-tiny-search'
METADATA_SIZE = 4096
NUM_PAGES = 76800
@@ -117,17 +116,19 @@ class TinyIndex(Generic[T]):
"""
Get the page at index i, decompress and deserialise it using JSON
"""
+ results = self._get_page_tuples(i)
+ return [self.item_factory(*item) for item in results]
+
+ def _get_page_tuples(self, i):
page_data = self.mmap[i * self.page_size:(i + 1) * self.page_size]
decompressed_data = self.decompressor.decompress(page_data)
- results = json.loads(decompressed_data.decode('utf8'))
- converted = [self.item_factory(*item) for item in results]
- return converted
+ return json.loads(decompressed_data.decode('utf8'))
def index(self, key: str, value: T):
assert type(value) == self.item_factory, f"Can only index the specified type" \
f" ({self.item_factory.__name__})"
page_index = self._get_key_page_index(key)
- current_page = self.get_page(page_index)
+ current_page = self._get_page_tuples(page_index)
if current_page is None:
current_page = []
value_tuple = astuple(value)
@@ -151,7 +152,7 @@ class TinyIndex(Generic[T]):
@staticmethod
def create(item_factory: Callable[..., T], index_path: str, num_pages: int, page_size: int):
if os.path.isfile(index_path):
- raise FileExistsError("Index file already exists")
+ raise FileExistsError(f"Index file '{index_path}' already exists")
metadata = TinyIndexMetadata(VERSION, page_size, num_pages, item_factory.__name__)
metadata_bytes = metadata.to_bytes()