Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Don't allow the same compressor/decompressor to be used from multiple threads

Daoud Clarke87825125e6ed
diff --git a/analyse/index_concurrency.py b/analyse/index_concurrency.py
new file mode 100644
index 0000000..3084101
--- /dev/null
+++ b/analyse/index_concurrency.py
@@ -0,0 +1,61 @@
+import multiprocessing
+import os
+import string
+from multiprocessing.pool import ThreadPool
+from pathlib import Path
+from random import Random
+
+from mwmbl.indexer.paths import INDEX_NAME
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document
+
+random = Random(2)
+
+
+def random_string():
+ randomly_generated_string = "".join([random.choice(string.ascii_letters) for _ in range(500)])
+ return randomly_generated_string
+
+
+DOCUMENTS = [
+ Document(title=f"Document {i}", url=f"https://something.com/{i}.html", extract=random_string(), score=i) for
+ i in range(1000)]
+
+PAGES = [random.randint(0, 10240000) for _ in range(20)]
+
+NUM_PAGES = 10240000
+PAGE_SIZE = 4096
+
+INDEX_PATH = Path(os.environ["HOME"] + "/mwmbl-data") / INDEX_NAME
+
+# INDEX_PATH = 'temp-index.tinysearch'
+
+try:
+ TinyIndex.create(Document, str(INDEX_PATH), num_pages=NUM_PAGES, page_size=PAGE_SIZE)
+except FileExistsError:
+ pass
+
+indexer = TinyIndex(Document, str(INDEX_PATH), 'r')
+indexer.__enter__()
+
+
+def read_page(page_index: int):
+ try:
+ page = indexer.get_page(page_index)
+ except Exception as e:
+ print(e)
+ return
+
+
+def index_reading_writing_multithreading():
+ # with multiprocessing.Pool(processes=10) as pool:
+ with ThreadPool(processes=10) as pool:
+ print("Start")
+ pool.map_async(read_page, [random.randint(0, indexer.num_pages) for i in range(1000000)])
+
+ pool.close()
+ pool.join()
+ print("End")
+
+
+if __name__ == '__main__':
+ index_reading_writing_multithreading()
diff --git a/analyse/stress_test_request.py b/analyse/stress_test_request.py
new file mode 100644
index 0000000..9eb2ff1
--- /dev/null
+++ b/analyse/stress_test_request.py
@@ -0,0 +1,29 @@
+"""
+Create a bunch of requests to the back end to stress test it.
+"""
+import multiprocessing
+from random import Random
+
+import requests
+
+random = Random(1)
+
+
+def random_query(query):
+ url = f"http://localhost:5000/api/v1/search/?s={query}"
+ print(url)
+ response = requests.get(url)
+ data = response.json()
+ # print(data)
+
+
+def run():
+ print("Start")
+ queries = ["".join(random.sample("the quick brown fox jumps over the lazy dog", 4)) for i in range(10000)]
+ with multiprocessing.Pool(processes=20) as pool:
+ pool.map(random_query, queries)
+ print("End")
+
+
+if __name__ == '__main__':
+ run()
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index 8b00655..d12cfd6 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -120,7 +120,8 @@ def _trim_items_to_page(compressor: ZstdCompressor, page_size: int, items:list[T
return _binary_search_fitting_size(compressor, page_size, items, 0, len(items))
-def _get_page_data(compressor: ZstdCompressor, page_size: int, items: list[T]):
+def _get_page_data(page_size: int, items: list[T]):
+ compressor = ZstdCompressor()
num_fitting, serialised_data = _trim_items_to_page(compressor, page_size, items)
compressed_data = compressor.compress(json.dumps(items[:num_fitting]).encode('utf8'))
@@ -157,8 +158,6 @@ class TinyIndex(Generic[T]):
self.num_pages = metadata.num_pages
self.page_size = metadata.page_size
- self.compressor = ZstdCompressor()
- self.decompressor = ZstdDecompressor()
logger.info(f"Loaded index with {self.num_pages} pages and {self.page_size} page size")
self.index_file = None
self.mmap = None
@@ -191,8 +190,9 @@ class TinyIndex(Generic[T]):
def _get_page_tuples(self, i):
page_data = self.mmap[i * self.page_size + METADATA_SIZE:(i + 1) * self.page_size + METADATA_SIZE]
+ decompressor = ZstdDecompressor()
try:
- decompressed_data = self.decompressor.decompress(page_data)
+ decompressed_data = decompressor.decompress(page_data)
except ZstdError as e:
logger.exception(f"Error decompressing page {i}: {e}")
return []
@@ -210,7 +210,7 @@ class TinyIndex(Generic[T]):
if self.mode != 'w':
raise UnsupportedOperation("The file is open in read mode, you cannot write")
- page_data = _get_page_data(self.compressor, self.page_size, data)
+ page_data = _get_page_data(self.page_size, data)
logger.debug(f"Got page data of length {len(page_data)}")
self.mmap[i * self.page_size + METADATA_SIZE:(i+1) * self.page_size + METADATA_SIZE] = page_data
@@ -223,8 +223,7 @@ class TinyIndex(Generic[T]):
metadata_bytes = metadata.to_bytes()
metadata_padded = _pad_to_page_size(metadata_bytes, METADATA_SIZE)
- compressor = ZstdCompressor()
- page_bytes = _get_page_data(compressor, page_size, [])
+ page_bytes = _get_page_data(page_size, [])
with open(index_path, 'wb') as index_file:
index_file.write(metadata_padded)
diff --git a/test/test_indexer.py b/test/test_indexer.py
index 6c1705f..06de1df 100644
--- a/test/test_indexer.py
+++ b/test/test_indexer.py
@@ -63,7 +63,7 @@ def test_get_page_data_single_doc():
# Compare the trimmed data to the actual data we're persisting
# We need to pad the trimmmed data, then it should be equal to the data we persist
padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)
- serialized_data = _get_page_data(compressor,page_size,items)
+ serialized_data = _get_page_data(page_size, items)
assert serialized_data == padded_trimmed_data
@@ -87,7 +87,7 @@ def test_get_page_data_many_docs_all_fit():
# Compare the trimmed data to the actual data we're persisting
# We need to pad the trimmed data, then it should be equal to the data we persist
- serialized_data = _get_page_data(compressor,page_size,items)
+ serialized_data = _get_page_data(page_size, items)
padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)
assert serialized_data == padded_trimmed_data
@@ -113,7 +113,7 @@ def test_get_page_data_many_docs_subset_fit():
# Compare the trimmed data to the actual data we're persisting
# We need to pad the trimmed data, then it should be equal to the data we persist
- serialized_data = _get_page_data(compressor,page_size,items)
+ serialized_data = _get_page_data(page_size, items)
padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)
assert serialized_data == padded_trimmed_data
\ No newline at end of file