An open source, non-profit web search engine
Don't allow the same compressor/decompressor to be used from multiple threads
Daoud Clarke87825125e6ed
diff --git a/analyse/index_concurrency.py b/analyse/index_concurrency.pynew file mode 100644index 0000000..3084101--- /dev/null+++ b/analyse/index_concurrency.py@@ -0,0 +1,61 @@+import multiprocessing+import os+import string+from multiprocessing.pool import ThreadPool+from pathlib import Path+from random import Random++from mwmbl.indexer.paths import INDEX_NAME+from mwmbl.tinysearchengine.indexer import TinyIndex, Document++random = Random(2)+++def random_string():+ randomly_generated_string = "".join([random.choice(string.ascii_letters) for _ in range(500)])+ return randomly_generated_string+++DOCUMENTS = [+ Document(title=f"Document {i}", url=f"https://something.com/{i}.html", extract=random_string(), score=i) for+ i in range(1000)]++PAGES = [random.randint(0, 10240000) for _ in range(20)]++NUM_PAGES = 10240000+PAGE_SIZE = 4096++INDEX_PATH = Path(os.environ["HOME"] + "/mwmbl-data") / INDEX_NAME++# INDEX_PATH = 'temp-index.tinysearch'++try:+ TinyIndex.create(Document, str(INDEX_PATH), num_pages=NUM_PAGES, page_size=PAGE_SIZE)+except FileExistsError:+ pass++indexer = TinyIndex(Document, str(INDEX_PATH), 'r')+indexer.__enter__()+++def read_page(page_index: int):+ try:+ page = indexer.get_page(page_index)+ except Exception as e:+ print(e)+ return+++def index_reading_writing_multithreading():+ # with multiprocessing.Pool(processes=10) as pool:+ with ThreadPool(processes=10) as pool:+ print("Start")+ pool.map_async(read_page, [random.randint(0, indexer.num_pages) for i in range(1000000)])++ pool.close()+ pool.join()+ print("End")+++if __name__ == '__main__':+ index_reading_writing_multithreading()diff --git a/analyse/stress_test_request.py b/analyse/stress_test_request.pynew file mode 100644index 0000000..9eb2ff1--- /dev/null+++ b/analyse/stress_test_request.py@@ -0,0 +1,29 @@+"""+Create a bunch of requests to the back end to stress test it.+"""+import multiprocessing+from random import Random++import requests++random = Random(1)+++def random_query(query):+ url = f"http://localhost:5000/api/v1/search/?s={query}"+ print(url)+ response = requests.get(url)+ data = response.json()+ # print(data)+++def run():+ print("Start")+ queries = ["".join(random.sample("the quick brown fox jumps over the lazy dog", 4)) for i in range(10000)]+ with multiprocessing.Pool(processes=20) as pool:+ pool.map(random_query, queries)+ print("End")+++if __name__ == '__main__':+ run()diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 8b00655..d12cfd6 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -120,7 +120,8 @@ def _trim_items_to_page(compressor: ZstdCompressor, page_size: int, items:list[Treturn _binary_search_fitting_size(compressor, page_size, items, 0, len(items))-def _get_page_data(compressor: ZstdCompressor, page_size: int, items: list[T]):+def _get_page_data(page_size: int, items: list[T]):+ compressor = ZstdCompressor()num_fitting, serialised_data = _trim_items_to_page(compressor, page_size, items)compressed_data = compressor.compress(json.dumps(items[:num_fitting]).encode('utf8'))@@ -157,8 +158,6 @@ class TinyIndex(Generic[T]):self.num_pages = metadata.num_pagesself.page_size = metadata.page_size- self.compressor = ZstdCompressor()- self.decompressor = ZstdDecompressor()logger.info(f"Loaded index with {self.num_pages} pages and {self.page_size} page size")self.index_file = Noneself.mmap = None@@ -191,8 +190,9 @@ class TinyIndex(Generic[T]):def _get_page_tuples(self, i):page_data = self.mmap[i * self.page_size + METADATA_SIZE:(i + 1) * self.page_size + METADATA_SIZE]+ decompressor = ZstdDecompressor()try:- decompressed_data = self.decompressor.decompress(page_data)+ decompressed_data = decompressor.decompress(page_data)except ZstdError as e:logger.exception(f"Error decompressing page {i}: {e}")return []@@ -210,7 +210,7 @@ class TinyIndex(Generic[T]):if self.mode != 'w':raise UnsupportedOperation("The file is open in read mode, you cannot write")- page_data = _get_page_data(self.compressor, self.page_size, data)+ page_data = _get_page_data(self.page_size, data)logger.debug(f"Got page data of length {len(page_data)}")self.mmap[i * self.page_size + METADATA_SIZE:(i+1) * self.page_size + METADATA_SIZE] = page_data@@ -223,8 +223,7 @@ class TinyIndex(Generic[T]):metadata_bytes = metadata.to_bytes()metadata_padded = _pad_to_page_size(metadata_bytes, METADATA_SIZE)- compressor = ZstdCompressor()- page_bytes = _get_page_data(compressor, page_size, [])+ page_bytes = _get_page_data(page_size, [])with open(index_path, 'wb') as index_file:index_file.write(metadata_padded)diff --git a/test/test_indexer.py b/test/test_indexer.pyindex 6c1705f..06de1df 100644--- a/test/test_indexer.py+++ b/test/test_indexer.py@@ -63,7 +63,7 @@ def test_get_page_data_single_doc():# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmmed data, then it should be equal to the data we persistpadded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)- serialized_data = _get_page_data(compressor,page_size,items)+ serialized_data = _get_page_data(page_size, items)assert serialized_data == padded_trimmed_data@@ -87,7 +87,7 @@ def test_get_page_data_many_docs_all_fit():# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmed data, then it should be equal to the data we persist- serialized_data = _get_page_data(compressor,page_size,items)+ serialized_data = _get_page_data(page_size, items)padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)assert serialized_data == padded_trimmed_data@@ -113,7 +113,7 @@ def test_get_page_data_many_docs_subset_fit():# Compare the trimmed data to the actual data we're persisting# We need to pad the trimmed data, then it should be equal to the data we persist- serialized_data = _get_page_data(compressor,page_size,items)+ serialized_data = _get_page_data(page_size, items)padded_trimmed_data = _pad_to_page_size(trimmed_data, page_size)assert serialized_data == padded_trimmed_data\ No newline at end of file